[chore](ci) collect core and adjust log dir

regression-test/pipeline/cloud_p0/deploy.sh

@@ -35,6 +35,7 @@ echo "#### Check env"
 if [[ -z "${teamcity_build_checkoutDir}" ]]; then echo "ERROR: env teamcity_build_checkoutDir not set" && exit 1; fi
 if [[ -z "${pr_num_from_trigger}" ]]; then echo "ERROR: env pr_num_from_trigger not set" && exit 1; fi
 if [[ -z "${commit_id_from_trigger}" ]]; then echo "ERROR: env commit_id_from_trigger not set" && exit 1; fi
+if [[ -z "${oss_ak}" || -z "${oss_sk}" ]]; then echo "ERROR: env oss_ak or oss_sk not set." && exit 1; fi
 
 echo "#### Deploy Doris ####"
 DORIS_HOME="${teamcity_build_checkoutDir}/output"

regression-test/pipeline/cloud_p0/prepare.sh

@@ -74,6 +74,7 @@ if [[ -z "${commit_id_from_trigger}" ]]; then echo "ERROR: env commit_id_from_tr
 if [[ -z "${commit_id_from_checkout}" ]]; then echo "ERROR: env commit_id_from_checkout not set" && exit 1; fi
 if [[ -z "${target_branch}" ]]; then echo "ERROR: env target_branch not set" && exit 1; fi
 if [[ -z "${cos_ak}" || -z "${cos_sk}" ]]; then echo "ERROR: env cos_ak or cos_sk not set" && exit 1; fi
+if [[ -z "${oss_ak}" || -z "${oss_sk}" ]]; then echo "ERROR: env oss_ak or oss_sk not set." && exit 1; fi
 
 echo "#### 1. check if need run"
 if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then

regression-test/pipeline/cloud_p0/run.sh

@@ -88,12 +88,12 @@ exit_flag="$?"
 echo "#### 5. check if need backup doris logs"
 if [[ ${exit_flag} != "0" ]]; then
     check_if_need_gcore
-    if file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_coredump.tar.gz"); then
-        upload_doris_log_to_oss "${file_name}"
-    fi
     stop_doris
     print_doris_fe_log
     print_doris_be_log
+    if file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_doris_coredump.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
     if file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
         upload_doris_log_to_oss "${file_name}"
     fi

regression-test/pipeline/common/doris-utils.sh

@@ -97,6 +97,7 @@ function start_doris_fe() {
     fi
     JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
     export JAVA_HOME
+    # export JACOCO_COVERAGE_OPT="-javaagent:/usr/local/jacoco/lib/jacocoagent.jar=excludes=org.apache.doris.thrift:org.apache.doris.proto:org.apache.parquet.format:com.aliyun*:com.amazonaws*:org.apache.hadoop.hive.metastore:org.apache.parquet.format,output=file,append=true,destfile=${DORIS_HOME}/fe/fe_cov.exec"
     "${DORIS_HOME}"/fe/bin/start_fe.sh --daemon
 
     if ! mysql --version >/dev/null; then sudo apt update && sudo apt install -y mysql-client; fi
@@ -122,6 +123,10 @@ function start_doris_be() {
     fi
     JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
     export JAVA_HOME
+    ASAN_SYMBOLIZER_PATH="$(command -v llvm-symbolizer)"
+    export ASAN_SYMBOLIZER_PATH
+    export ASAN_OPTIONS="symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:use_sigaltstack=0:detect_leaks=0:fast_unwind_on_malloc=0"
+    export TCMALLOC_SAMPLE_PARAMETER=524288
     sysctl -w vm.max_map_count=2000000 &&
         ulimit -n 200000 &&
         ulimit -c unlimited &&
@@ -446,66 +451,106 @@ function set_doris_session_variables_from_file() {
 
 archive_doris_logs() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
-    archive_name="$1"
+    local archive_name="$1"
     if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi
-    archive_content="fe/conf fe/log be/conf be/log"
-    if [[ -d "${DORIS_HOME}"/regression-test/log ]]; then
-        archive_content="${archive_content} regression-test/log"
-    fi
-    if [[ -f "${DORIS_HOME:-}"/session_variables ]]; then
-        archive_content="${archive_content} session_variables"
-    fi
-    if [[ -d "${DORIS_HOME}"/ms ]]; then
-        mkdir -p "${DORIS_HOME}"/foundationdb/log
-        cp -rf /var/log/foundationdb/* "${DORIS_HOME}"/foundationdb/log/
-        archive_content="${archive_content} ms/conf ms/log foundationdb/log"
-    fi
-    if [[ -d "${DORIS_HOME}"/recycler ]]; then
-        archive_content="${archive_content} recycler/conf recycler/log"
-    fi
-    if [[ -d "${DORIS_HOME}"/be/storage/error_log ]]; then
-        archive_content="${archive_content} be/storage/error_log"
-    fi
+    local archive_dir="${archive_name%.tar.gz}"
+    rm -rf "${DORIS_HOME:?}/${archive_dir}"
+    mkdir -p "${DORIS_HOME}/${archive_dir}"
+    (
+        cd "${DORIS_HOME}" || return 1
+        cp --parents -rf "fe/conf" "${archive_dir}"/
+        cp --parents -rf "fe/log" "${archive_dir}"/
+        cp --parents -rf "be/conf" "${archive_dir}"/
+        cp --parents -rf "be/log" "${archive_dir}"/
+        if [[ -d "${DORIS_HOME}"/regression-test/log ]]; then
+            # try to hide ak and sk
+            if sed -i "s/${cos_ak:-}//g;s/${cos_sk:-}//g" regression-test/log/* &>/dev/null; then :; fi
+            cp --parents -rf "regression-test/log" "${archive_dir}"/
+        fi
+        if [[ -f "${DORIS_HOME}"/session_variables ]]; then
+            cp --parents -rf "session_variables" "${archive_dir}"/
+        fi
+        if [[ -d "${DORIS_HOME}"/ms ]]; then
+            mkdir -p "${archive_dir}"/foundationdb/log
+            cp --parents -rf /var/log/foundationdb/* "${archive_dir}"/foundationdb/log/
+            cp --parents -rf "ms/conf" "${archive_dir}"/
+            cp --parents -rf "ms/log" "${archive_dir}"/
+        fi
+        if [[ -d "${DORIS_HOME}"/recycler ]]; then
+            cp --parents -rf "recycler/conf" "${archive_dir}"/
+            cp --parents -rf "recycler/log" "${archive_dir}"/
+        fi
+        if [[ -d "${DORIS_HOME}"/be/storage/error_log ]]; then
+            cp --parents -rf "be/storage/error_log" "${archive_dir}"/
+        fi
+    )
 
-    # shellcheck disable=SC2086
     if tar -I pigz \
         --directory "${DORIS_HOME}" \
         -cf "${DORIS_HOME}/${archive_name}" \
-        ${archive_content}; then
+        "${archive_dir}"; then
+        rm -rf "${DORIS_HOME:?}/${archive_dir}"
         echo "${DORIS_HOME}/${archive_name}"
     else
         return 1
     fi
 }
 
+wait_coredump_file_ready() {
+    # if the size of coredump file does not changed in 5 seconds, we think it has generated done
+    local coredump_file="$1"
+    if [[ -z "${coredump_file}" ]]; then echo "ERROR: coredump_file is required" && return 1; fi
+    initial_size=$(stat -c %s "${coredump_file}")
+    while true; do
+        sleep 5
+        current_size=$(stat -c %s "${coredump_file}")
+        if [[ ${initial_size} -eq ${current_size} ]]; then
+            break
+        else
+            initial_size=${current_size}
+        fi
+    done
+}
+
 archive_doris_coredump() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
     archive_name="$1"
+    COREDUMP_SIZE_THRESHOLD="${COREDUMP_SIZE_THRESHOLD:-85899345920}" # if coredump size over 80G, do not archive"
     if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi
-    be_pid="$(cat "${DORIS_HOME}"/be/bin/be.pid)"
-    if [[ -z "${be_id}" ]]; then echo "ERROR: can not find be id from ${DORIS_HOME}/be/bin/be.pid" && return 1; fi
-    if corename=$(find /var/lib/apport/coredump/ -type f -name "core.*${be_pid}.*"); then
-        initial_size=$(stat -c %s "${corename}")
-        while true; do
-            sleep 2
-            current_size=$(stat -c %s "${corename}")
-            if [[ ${initial_size} -eq ${current_size} ]]; then
-                break
-            else
-                initial_size=${current_size}
+    local archive_dir="${archive_name%.tar.gz}"
+    rm -rf "${DORIS_HOME:?}/${archive_dir}"
+    mkdir -p "${DORIS_HOME}/${archive_dir}"
+    declare -A pids
+    pids['be']="$(cat "${DORIS_HOME}"/be/bin/be.pid)"
+    pids['ms']="$(cat "${DORIS_HOME}"/ms/bin/doris_cloud.pid)"
+    pids['recycler']="$(cat "${DORIS_HOME}"/recycler/bin/doris_cloud.pid)"
+    for p in "${!pids[@]}"; do
+        pid="${pids[${p}]}"
+        if [[ -z "${pid}" ]]; then continue; fi
+        if coredump_file=$(find /var/lib/apport/coredump/ -type f -name "core.*${pid}.*") &&
+            wait_coredump_file_ready "${coredump_file}"; then
+            file_size=$(stat -c %s "${coredump_file}")
+            if ((file_size <= COREDUMP_SIZE_THRESHOLD)); then
+                mkdir -p "${DORIS_HOME}/${archive_dir}/${p}"
+                if [[ "${p}" == "be" ]]; then
+                    mv "${DORIS_HOME}"/be/lib/doris_be "${DORIS_HOME}/${archive_dir}/${p}"
+                elif [[ "${p}" == "ms" ]]; then
+                    mv "${DORIS_HOME}"/ms/lib/doris_cloud "${DORIS_HOME}/${archive_dir}/${p}"
+                elif [[ "${p}" == "recycler" ]]; then
+                    mv "${DORIS_HOME}"/recycler/lib/doris_cloud "${DORIS_HOME}/${archive_dir}/${p}"
+                fi
+                mv "${coredump_file}" "${DORIS_HOME}/${archive_dir}/${p}"
             fi
-        done
-        file_size=$(stat -c %s "${corename}")
-        if ((file_size > 85899345920)); then
-            echo "coredump size ${file_size} over 80G, not upload"
-            return 1
-        else
-            #压缩core文件
-            mv "${corename}" "${DORIS_HOME}"/be/lib/
-            cd "${DORIS_HOME}"/be/lib/ || return 1
-            tar -I pigz -cf core.tar.gz be/lib/doris_be "be/lib/$(basename "${corename}")" >/dev/null
-            echo "$(pwd)/core.tar.gz"
         fi
+    done
+
+    if tar -I pigz \
+        --directory "${DORIS_HOME}" \
+        -cf "${DORIS_HOME}/${archive_name}" \
+        "${archive_dir}"; then
+        echo "${DORIS_HOME}/${archive_name}"
+    else
+        return 1
     fi
 }
 

regression-test/pipeline/common/oss-utils.sh

@@ -18,22 +18,22 @@
 
 function install_ossutil() {
     if command -v ossutil >/dev/null; then return 0; fi
-    if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then
-        echo "ERROR: env OSS_accessKeyID or OSS_accessKeySecret not set."
+    if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then
+        echo "ERROR: env oss_ak or oss_sk not set."
         return 1
     fi
     curl https://gosspublic.alicdn.com/ossutil/install.sh | sudo bash
     echo "[Credentials]
 language=EN
 endpoint=oss-cn-hongkong-internal.aliyuncs.com
-accessKeyID=${OSS_accessKeyID:-}
-accessKeySecret=${OSS_accessKeySecret:-}
+accessKeyID=${oss_ak:-}
+accessKeySecret=${oss_sk:-}
 " >~/.ossutilconfig
 }
 
 function check_oss_file_exist() {
-    if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then
-        echo "ERROR: env OSS_accessKeyID and OSS_accessKeySecret not set"
+    if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then
+        echo "ERROR: env oss_ak and oss_sk not set"
         return 1
     fi
     # Check if the file exists.
@@ -42,8 +42,8 @@ function check_oss_file_exist() {
     OSS_DIR="${OSS_DIR:-"oss://opensource-pipeline/compile-release"}"
     install_ossutil
     if ossutil stat \
-        -i "${OSS_accessKeyID}" \
-        -k "${OSS_accessKeySecret}" \
+        -i "${oss_ak}" \
+        -k "${oss_sk}" \
         "${OSS_DIR}/${file_name}"; then
         echo "INFO: ${file_name} file exists." && return 0
     else
@@ -58,8 +58,8 @@ function download_oss_file() {
     OSS_DIR="${OSS_DIR:-"oss://opensource-pipeline/compile-release"}"
     install_ossutil
     if ossutil cp -f \
-        -i "${OSS_accessKeyID}" \
-        -k "${OSS_accessKeySecret}" \
+        -i "${oss_ak}" \
+        -k "${oss_sk}" \
         "${OSS_DIR}/${file_name}" \
         "${file_name}"; then
         echo "INFO: download ${file_name} success" && return 0
@@ -69,8 +69,8 @@ function download_oss_file() {
 }
 
 function upload_file_to_oss() {
-    if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then
-        echo "ERROR: env OSS_accessKeyID and OSS_accessKeySecret not set"
+    if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then
+        echo "ERROR: env oss_ak and oss_sk not set"
         return 1
     fi
     if [[ ! -f "$1" ]] || [[ "$1" != "/"* ]]; then
@@ -87,8 +87,8 @@ function upload_file_to_oss() {
     install_ossutil
     cd "${dir_name}" || return 1
     if ossutil cp -f \
-        -i "${OSS_accessKeyID}" \
-        -k "${OSS_accessKeySecret}" \
+        -i "${oss_ak}" \
+        -k "${oss_sk}" \
         "${file_name}" \
         "${OSS_DIR}/${file_name}"; then
         if ! check_oss_file_exist "${file_name}"; then return 1; fi