[fix](cloud) provide a conf to enable/disable streamload commit on be (pick #37848)

.asf.yaml

@@ -61,11 +61,15 @@ github:
           - ShellCheck          
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
+          - Build Third Party Libraries (macOS-arm64)
           - COMPILE (DORIS_COMPILE)
           - Need_2_Approval
+          - Cloud UT (Doris Cloud UT)
+          - performance (Doris Performance)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true
+        require_code_owner_reviews: true
         required_approving_review_count: 1
     branch-1.1-lts:
       required_status_checks:
@@ -78,6 +82,28 @@ github:
         dismiss_stale_reviews: true
         required_approving_review_count: 1
 
+    branch-2.1:
+      required_status_checks:
+        # strict means "Require branches to be up to date before merging".
+        strict: false
+        contexts:
+          - License Check
+          - Clang Formatter
+          - CheckStyle
+          - Build Broker
+          - ShellCheck
+          - Build Third Party Libraries (Linux)
+          - Build Third Party Libraries (macOS)
+          - COMPILE (DORIS_COMPILE)
+          - External Regression (Doris External Regression)
+          - FE UT (Doris FE UT)
+          - BE UT (Doris BE UT)
+          - P0 Regression (Doris Regression)
+
+        required_pull_request_reviews:
+          dismiss_stale_reviews: true
+          required_approving_review_count: 1
+
     branch-2.0:
       required_status_checks:
       # if strict is true, means "Require branches to be up to date before merging".

.github/CODEOWNERS

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+be/src/io/* @platoneko @gavinchou @dataroaring
+fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,3 @@ Issue Number: close #xxx
 
 <!--Describe your changes.-->
 
-## Further comments
-
-If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc...
-

.github/workflows/be-ut-mac.yml

@@ -53,13 +53,14 @@ jobs:
         uses: ./.github/actions/ccache-action
         with:
           key: BE-UT-macOS
-          max-size: "2G"
+          max-size: "5G"
           restore-keys: BE-UT-macOS-
 
       - name: Run UT ${{ github.ref }}
         if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }}
         run: |
           cellars=(
+            'm4'
             'automake'
             'autoconf'
             'libtool'
@@ -95,4 +96,5 @@ jobs:
           tar -xvf doris-thirdparty-prebuilt-darwin-x86_64.tar.xz
           popd
 
+          export JAVA_HOME="${JAVA_HOME_17_X64%\/}"
           ./run-be-ut.sh --run -j "$(nproc)" --clean

.github/workflows/build-thirdparty.yml

@@ -32,7 +32,7 @@ jobs:
       thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
     steps:
       - name: Checkout ${{ github.ref }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           submodules: recursive
@@ -67,7 +67,7 @@ jobs:
             remove-docker-images: 'true'
 
       - name: Checkout ${{ github.ref }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download
         run: |
@@ -129,10 +129,10 @@ jobs:
     name: Build Third Party Libraries (macOS)
     needs: changes
     if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
-    runs-on: macos-12
+    runs-on: macos-13
     steps:
       - name: Checkout ${{ github.ref }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download
         run: |
@@ -144,6 +144,58 @@ jobs:
       - name: Prepare
         run: |
           packages=(
+            'm4'
+            'automake'
+            'autoconf'
+            'libtool'
+            'pkg-config'
+            'texinfo'
+            'coreutils'
+            'gnu-getopt'
+            'python@3'
+            'cmake'
+            'ninja'
+            'ccache'
+            'bison'
+            'byacc'
+            'gettext'
+            'wget'
+            'pcre'
+            'openjdk@11'
+            'maven'
+            'node'
+            'llvm@16'
+          )
+
+          brew install "${packages[@]}" || true
+
+      - name: Build
+        run: |
+          export MACOSX_DEPLOYMENT_TARGET=12.0
+
+          cd thirdparty
+          ./build-thirdparty.sh -j "$(nproc)"
+
+  build_macos_arm64:
+    name: Build Third Party Libraries (macOS-arm64)
+    needs: changes
+    if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
+    runs-on: macos-14
+    steps:
+      - name: Checkout ${{ github.ref }}
+        uses: actions/checkout@v4
+
+      - name: Download
+        run: |
+          cd thirdparty
+          curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \
+            -o doris-thirdparty-source.tgz
+          tar -zxvf doris-thirdparty-source.tgz
+
+      - name: Prepare
+        run: |
+          packages=(
+            'm4'
             'automake'
             'autoconf'
             'libtool'

.github/workflows/comment-to-trigger-teamcity.yml

@@ -111,7 +111,7 @@ jobs:
             echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT"
           fi
           if file_changed_performance; then
-            echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
           else
             echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT"
           fi
@@ -149,15 +149,22 @@ jobs:
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
-              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_fe_ut }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "feut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0), skip run feut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run feut"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "feut" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         fi
 
 
@@ -167,15 +174,22 @@ jobs:
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
-              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_be_ut }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "beut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0), skip run beut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run beut"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "beut" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         fi
 
     - name: "Trigger or Skip cloudut"

.gitignore

@@ -131,5 +131,13 @@ lru_cache_test
 /fe/fe-core/src/test/resources/real-help-resource.zip
 /ui/dist
 
+# docker
+docker/thirdparties/docker-compose/*/data
+docker/thirdparties/docker-compose/*/logs
+docker/thirdparties/docker-compose/*/*.yaml
+docker/runtime/be/resource/apache-doris/
+
 # other
 compile_commands.json
+.github
+

.licenserc.yaml

@@ -82,6 +82,7 @@ header:
     - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql"
     - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
     - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
+    - "docker/thirdparties/docker-compose/hive/scripts/data/**"
     - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
     - "conf/mysql_ssl_default_certificate/*"
     - "conf/mysql_ssl_default_certificate/client_certificate/ca.pem"

LICENSE.txt

@@ -718,3 +718,11 @@ LICENSE file in the root directory of this source tree. An additional grant
 of patent rights can be found in the PATENTS file in the same directory.
 
 ----------------------------------------------------------------------------------
+
+me.bechberger.ap-loader-all: Apache 2.0 license
+
+Apache 2.0, Copyright 2023 SAP SE or an SAP affiliate company, Johannes Bechberger and ap-loader contributors
+
+This project is maintained by the SapMachine team at SAP SE
+
+----------------------------------------------------------------------------------

README.md

@@ -26,19 +26,15 @@ under the License.
 [![GitHub release](https://img.shields.io/github/release/apache/doris.svg)](https://github.com/apache/doris/releases)
 [![Jenkins Vec](https://img.shields.io/jenkins/tests?compact_message&jobUrl=https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized&label=VectorizedEngine)](https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized)
 [![Total Lines](https://tokei.rs/b1/github/apache/doris?category=lines)](https://github.com/apache/doris)
-[![Join the Doris Community at Slack](https://img.shields.io/badge/chat-slack-brightgreen)](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-28il1o2wk-DD6LsLOz3v4aD92Mu0S0aQ)
-[![Join the chat at https://gitter.im/apache-doris/Lobby](https://badges.gitter.im/apache-doris/Lobby.svg)](https://gitter.im/apache-doris/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Join the Doris Community on Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-2kl08hzc0-SPJe4VWmL_qzrFd2u2XYQA)
 [![EN doc](https://img.shields.io/badge/Docs-English-blue.svg)](https://doris.apache.org/docs/get-starting/quick-start)
 [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)]([https://doris.apache.org/zh-CN/docs/dev/get-starting/what-is-apache-doris](https://doris.apache.org/zh-CN/docs/get-starting/what-is-apache-doris))
 
-Apache Doris is an easy-to-use, high-performance and real-time analytical database based on MPP architecture, known for its extreme speed and ease of use. It only requires a sub-second response time to return query results under massive data and can support not only high-concurrent point query scenarios but also high-throughput complex analysis scenarios.
+Apache Doris is an MPP-based real-time data warehouse known for its high query speed. For queries on large datasets, it returns results in sub-seconds. It supports both high-concurrency point queries and high-throughput complex analysis. It can be used for report analysis, ad-hoc queries, unified data warehouse building, and data lake query acceleration. Based on Apache Doris, users can build applications for user behavior analysis, A/B testing platform, log analysis, and e-commerce order analysis.
 
-All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
+Please visit our [official download page](https://doris.apache.org/download/) to get the latest release version.
 
-🎉 Version 2.1.1 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.1) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
-
-
-🎉 Version 2.0.8 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.0.8) here. 
+The current stable version is the 2.0.x series, and the latest version is the 2.1.x series. For production, it is recommended to use the latest version of the 2.0.x series. And if used for POC or testing, it is recommended to use the latest version of the 2.1.x series.
 
 👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
 
@@ -137,7 +133,7 @@ In terms of optimizers, Doris uses a combination of CBO and RBO. RBO supports co
 
 **Apache Doris has graduated from Apache incubator successfully and become a Top-Level Project in June 2022**. 
 
-Currently, the Apache Doris community has gathered more than 400 contributors from nearly 200 companies in different industries, and the number of active contributors is close to 100 per month.
+Currently, the Apache Doris community has gathered more than 600 contributors from over 200 companies in different industries, and the number of monthly active contributors exceeds 100.
 
 
 [![Monthly Active Contributors](https://contributor-overtime-api.apiseven.com/contributors-svg?chart=contributorMonthlyActivity&repo=apache/doris)](https://www.apiseven.com/en/contributor-graph?chart=contributorMonthlyActivity&repo=apache/doris)
@@ -216,7 +212,7 @@ Contact us through the following mailing list.
 
 * Apache Doris Official Website - [Site](https://doris.apache.org)
 * Developer Mailing list - <dev@doris.apache.org>. Mail to <dev-subscribe@doris.apache.org>, follow the reply to subscribe the mail list.
-* Slack channel - [Join the Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-28il1o2wk-DD6LsLOz3v4aD92Mu0S0aQ)
+* Slack channel - [Join the Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-2kl08hzc0-SPJe4VWmL_qzrFd2u2XYQA)
 * Twitter - [Follow @doris_apache](https://twitter.com/doris_apache)