[chore](sql) Forbid show hidden columns and create table with hidden column (#38796)

.asf.yaml

@@ -52,15 +52,12 @@ github:
           - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
-          - P0 Regression PipelineX (Doris Regression)
           - P1 Regression (Doris Regression)
           - External Regression (Doris External Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
-          - Build Documents
           - ShellCheck          
-          - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
@@ -95,8 +92,6 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - ShellCheck
-          - Build Documents
-          - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
@@ -109,7 +104,6 @@ github:
     - LemonLiTree
     - Yukang-Lian
     - TangSiyang2001
-    - platoneko
     - Lchangliang
     - freemandealer
     - shuke987

.clang-tidy

@@ -4,6 +4,10 @@ Checks: |
   clang-diagnostic-*,
   clang-analyzer-*,
   bugprone-redundant-branch-condition,
+  bugprone-use-after-move,
+  bugprone-bool-pointer-implicit-conversion,
+  bugprone-unused-raii,
+  bugprone-fold-init-type,
   modernize-*,
   -modernize-use-trailing-return-type,
   -modernize-use-nodiscard,
@@ -19,15 +23,16 @@ Checks: |
   -readability-inconsistent-declaration-parameter-name,
   -readability-isolate-declaration,
   -readability-named-parameter,
+  -readability-avoid-const-params-in-decls,
+  -readability-convert-member-functions-to-static,
   portability-simd-intrinsics,
   performance-type-promotion-in-math-fn,
   performance-faster-string-find,
   performance-inefficient-algorithm,
   performance-move-const-arg
-WarningsAsErrors: '*'
+WarningsAsErrors: "*"
 CheckOptions:
-  - key:             readability-function-size.LineThreshold
-    value:           '80'
-  - key:             readability-function-cognitive-complexity.Threshold
-    value:           '50'
-
+  - key: readability-function-size.LineThreshold
+    value: "80"
+  - key: readability-function-cognitive-complexity.Threshold
+    value: "50"

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,3 @@ Issue Number: close #xxx
 
 <!--Describe your changes.-->
 
-## Further comments
-
-If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc...
-

.github/workflows/build-thirdparty.yml

@@ -166,10 +166,12 @@ jobs:
             'llvm@16'
           )
 
-          brew install "${packages[@]}"
+          brew install "${packages[@]}" || true
 
       - name: Build
         run: |
+          export MACOSX_DEPLOYMENT_TARGET=12.0
+
           cd thirdparty
           ./build-thirdparty.sh -j "$(nproc)"
 

.github/workflows/comment-to-trigger-teamcity.yml

@@ -45,9 +45,7 @@ jobs:
             "${COMMENT_BODY}" == *'run p1'* ||
             "${COMMENT_BODY}" == *'run external'* ||
             "${COMMENT_BODY}" == *'run pipelinex_p0'* ||
-            "${COMMENT_BODY}" == *'run clickbench'* ||
             "${COMMENT_BODY}" == *'run arm'* ||
-            "${COMMENT_BODY}" == *'run tpch'* ||
             "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
         else
@@ -64,7 +62,7 @@ jobs:
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
 
-        reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch|performance)( [1-9]*[0-9]+)*"
+        reg="run (buildall|compile|p0|p1|feut|beut|external|pipelinex_p0|arm|performance)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
@@ -91,13 +89,6 @@ jobs:
           else
             echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT"
           fi
-          if file_changed_ckb; then
-            echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT"
-          else
-            echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_tpch=false" | tee -a "$GITHUB_OUTPUT"
-          fi
           if file_changed_regression_p0; then
             echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
             echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
@@ -123,8 +114,6 @@ jobs:
           echo "INFO: failed to _get_pr_changed_files, default trigger all"
           echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
@@ -167,8 +156,13 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
+        if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then
+          changed="true"
+        else
+          changed="false"
+        fi
         trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_p0 }}" \
+          "${changed}" \
           "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
           "compile" \
@@ -247,52 +241,40 @@ jobs:
           "arm" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip clickbench"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
-      run: |
-        source ./regression-test/pipeline/common/teamcity-utils.sh
-        set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_ckb }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "clickbench" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
-
-    - name: "Trigger or Skip tpch"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
-      run: |
-        source ./regression-test/pipeline/common/teamcity-utils.sh
-        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
-          echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" && exit
-        fi
-        set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_tpch }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "tpch" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
-
     - name: "Trigger or Skip performance"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
-          echo "PR target branch is master, need run performance"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+          echo "PR target branch in (master, branch-2.0), need run performance"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_performance }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "performance" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_performance }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "perf" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not master, skip run performance"
+          echo "PR target branch not in (master, branch-2.0), skip run performance"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "performance" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "perf" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         fi

.gitignore

@@ -54,10 +54,14 @@ thirdparty/installed*
 thirdparty/doris-thirdparty*.tar.xz
 
 docker/thirdparties/docker-compose/mysql/data
+docker/thirdparties/docker-compose/hive/scripts/tpch1.db/
+docker/thirdparties/docker-compose/hive/scripts/paimon1
 
 fe_plugins/output
 fe_plugins/**/.factorypath
 
+docker/thirdparties/docker-compose/hive/scripts/data/*/*/data
+
 fs_brokers/apache_hdfs_broker/src/main/resources/
 fs_brokers/apache_hdfs_broker/src/main/thrift/
 
@@ -79,6 +83,7 @@ samples/doris-demo/remote-udf-cpp-demo/function_server_demo
 
 # FE
 fe/mocked/
+fe/gen/
 fe/fe-core/gen/
 fe/fe-core/src/main/resources/static/
 fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.tokens
@@ -97,7 +102,6 @@ be/tags
 be/test/olap/test_data/tablet_meta_test.hdr
 be/.devcontainer/
 be/src/apache-orc/
-zoneinfo/
 
 ## tools
 tools/ssb-tools/ssb-data/

.gitmodules

@@ -24,6 +24,3 @@
 	path = be/src/clucene
 	url = https://github.com/apache/doris-thirdparty.git
 	branch = clucene
-[submodule ".github/actions/create-or-update-comment"]
-	path = .github/actions/create-or-update-comment
-	url = https://github.com/peter-evans/create-or-update-comment.git

.licenserc.yaml

@@ -40,6 +40,7 @@ header:
     - "**/*.sql"
     - "**/*.lock"
     - "**/*.out"
+    - "**/*.parquet"
     - "docs/.markdownlintignore"
     - "fe/fe-core/src/test/resources/data/net_snmp_normal"
     - "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaLexer.g4"
@@ -77,11 +78,7 @@ header:
     - "docs/package-lock.json"
     - "regression-test/script/README"
     - "regression-test/suites/load_p0/stream_load/data"
-    - "docker/thirdparties/docker-compose/hive/scripts/README"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
+    - "docker/thirdparties/docker-compose/hive/scripts/**"
     - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
     - "conf/mysql_ssl_default_certificate/*"
     - "conf/mysql_ssl_default_certificate/client_certificate/ca.pem"

LICENSE.txt

@@ -718,3 +718,11 @@ LICENSE file in the root directory of this source tree. An additional grant
 of patent rights can be found in the PATENTS file in the same directory.
 
 ----------------------------------------------------------------------------------
+
+me.bechberger.ap-loader-all: Apache 2.0 license
+
+Apache 2.0, Copyright 2023 SAP SE or an SAP affiliate company, Johannes Bechberger and ap-loader contributors
+
+This project is maintained by the SapMachine team at SAP SE
+
+----------------------------------------------------------------------------------