diff --git a/regression-test/pipeline/cloud_p0/deploy.sh b/regression-test/pipeline/cloud_p0/deploy.sh index 3be6c85f185561..2cfe9f96f480d2 100644 --- a/regression-test/pipeline/cloud_p0/deploy.sh +++ b/regression-test/pipeline/cloud_p0/deploy.sh @@ -35,6 +35,7 @@ echo "#### Check env" if [[ -z "${teamcity_build_checkoutDir}" ]]; then echo "ERROR: env teamcity_build_checkoutDir not set" && exit 1; fi if [[ -z "${pr_num_from_trigger}" ]]; then echo "ERROR: env pr_num_from_trigger not set" && exit 1; fi if [[ -z "${commit_id_from_trigger}" ]]; then echo "ERROR: env commit_id_from_trigger not set" && exit 1; fi +if [[ -z "${oss_ak}" || -z "${oss_sk}" ]]; then echo "ERROR: env oss_ak or oss_sk not set." && exit 1; fi echo "#### Deploy Doris ####" DORIS_HOME="${teamcity_build_checkoutDir}/output" diff --git a/regression-test/pipeline/cloud_p0/prepare.sh b/regression-test/pipeline/cloud_p0/prepare.sh index 195e9c01b5b7d6..25a7d9f3d7430e 100644 --- a/regression-test/pipeline/cloud_p0/prepare.sh +++ b/regression-test/pipeline/cloud_p0/prepare.sh @@ -74,6 +74,7 @@ if [[ -z "${commit_id_from_trigger}" ]]; then echo "ERROR: env commit_id_from_tr if [[ -z "${commit_id_from_checkout}" ]]; then echo "ERROR: env commit_id_from_checkout not set" && exit 1; fi if [[ -z "${target_branch}" ]]; then echo "ERROR: env target_branch not set" && exit 1; fi if [[ -z "${cos_ak}" || -z "${cos_sk}" ]]; then echo "ERROR: env cos_ak or cos_sk not set" && exit 1; fi +if [[ -z "${oss_ak}" || -z "${oss_sk}" ]]; then echo "ERROR: env oss_ak or oss_sk not set." && exit 1; fi echo "#### 1. check if need run" if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh index fe27ad08f8802e..4d2094fee197f5 100644 --- a/regression-test/pipeline/cloud_p0/run.sh +++ b/regression-test/pipeline/cloud_p0/run.sh @@ -88,12 +88,12 @@ exit_flag="$?" echo "#### 5. check if need backup doris logs" if [[ ${exit_flag} != "0" ]]; then check_if_need_gcore - if file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_coredump.tar.gz"); then - upload_doris_log_to_oss "${file_name}" - fi stop_doris print_doris_fe_log print_doris_be_log + if file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_doris_coredump.tar.gz"); then + upload_doris_log_to_oss "${file_name}" + fi if file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then upload_doris_log_to_oss "${file_name}" fi diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh index 719df14790cbd2..b28d72fb69d3fd 100644 --- a/regression-test/pipeline/common/doris-utils.sh +++ b/regression-test/pipeline/common/doris-utils.sh @@ -97,6 +97,7 @@ function start_doris_fe() { fi JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')" export JAVA_HOME + # export JACOCO_COVERAGE_OPT="-javaagent:/usr/local/jacoco/lib/jacocoagent.jar=excludes=org.apache.doris.thrift:org.apache.doris.proto:org.apache.parquet.format:com.aliyun*:com.amazonaws*:org.apache.hadoop.hive.metastore:org.apache.parquet.format,output=file,append=true,destfile=${DORIS_HOME}/fe/fe_cov.exec" "${DORIS_HOME}"/fe/bin/start_fe.sh --daemon if ! mysql --version >/dev/null; then sudo apt update && sudo apt install -y mysql-client; fi @@ -122,6 +123,10 @@ function start_doris_be() { fi JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')" export JAVA_HOME + ASAN_SYMBOLIZER_PATH="$(command -v llvm-symbolizer)" + export ASAN_SYMBOLIZER_PATH + export ASAN_OPTIONS="symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:use_sigaltstack=0:detect_leaks=0:fast_unwind_on_malloc=0" + export TCMALLOC_SAMPLE_PARAMETER=524288 sysctl -w vm.max_map_count=2000000 && ulimit -n 200000 && ulimit -c unlimited && @@ -446,66 +451,106 @@ function set_doris_session_variables_from_file() { archive_doris_logs() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi - archive_name="$1" + local archive_name="$1" if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi - archive_content="fe/conf fe/log be/conf be/log" - if [[ -d "${DORIS_HOME}"/regression-test/log ]]; then - archive_content="${archive_content} regression-test/log" - fi - if [[ -f "${DORIS_HOME:-}"/session_variables ]]; then - archive_content="${archive_content} session_variables" - fi - if [[ -d "${DORIS_HOME}"/ms ]]; then - mkdir -p "${DORIS_HOME}"/foundationdb/log - cp -rf /var/log/foundationdb/* "${DORIS_HOME}"/foundationdb/log/ - archive_content="${archive_content} ms/conf ms/log foundationdb/log" - fi - if [[ -d "${DORIS_HOME}"/recycler ]]; then - archive_content="${archive_content} recycler/conf recycler/log" - fi - if [[ -d "${DORIS_HOME}"/be/storage/error_log ]]; then - archive_content="${archive_content} be/storage/error_log" - fi + local archive_dir="${archive_name%.tar.gz}" + rm -rf "${DORIS_HOME:?}/${archive_dir}" + mkdir -p "${DORIS_HOME}/${archive_dir}" + ( + cd "${DORIS_HOME}" || return 1 + cp --parents -rf "fe/conf" "${archive_dir}"/ + cp --parents -rf "fe/log" "${archive_dir}"/ + cp --parents -rf "be/conf" "${archive_dir}"/ + cp --parents -rf "be/log" "${archive_dir}"/ + if [[ -d "${DORIS_HOME}"/regression-test/log ]]; then + # try to hide ak and sk + if sed -i "s/${cos_ak:-}//g;s/${cos_sk:-}//g" regression-test/log/* &>/dev/null; then :; fi + cp --parents -rf "regression-test/log" "${archive_dir}"/ + fi + if [[ -f "${DORIS_HOME}"/session_variables ]]; then + cp --parents -rf "session_variables" "${archive_dir}"/ + fi + if [[ -d "${DORIS_HOME}"/ms ]]; then + mkdir -p "${archive_dir}"/foundationdb/log + cp --parents -rf /var/log/foundationdb/* "${archive_dir}"/foundationdb/log/ + cp --parents -rf "ms/conf" "${archive_dir}"/ + cp --parents -rf "ms/log" "${archive_dir}"/ + fi + if [[ -d "${DORIS_HOME}"/recycler ]]; then + cp --parents -rf "recycler/conf" "${archive_dir}"/ + cp --parents -rf "recycler/log" "${archive_dir}"/ + fi + if [[ -d "${DORIS_HOME}"/be/storage/error_log ]]; then + cp --parents -rf "be/storage/error_log" "${archive_dir}"/ + fi + ) - # shellcheck disable=SC2086 if tar -I pigz \ --directory "${DORIS_HOME}" \ -cf "${DORIS_HOME}/${archive_name}" \ - ${archive_content}; then + "${archive_dir}"; then + rm -rf "${DORIS_HOME:?}/${archive_dir}" echo "${DORIS_HOME}/${archive_name}" else return 1 fi } +wait_coredump_file_ready() { + # if the size of coredump file does not changed in 5 seconds, we think it has generated done + local coredump_file="$1" + if [[ -z "${coredump_file}" ]]; then echo "ERROR: coredump_file is required" && return 1; fi + initial_size=$(stat -c %s "${coredump_file}") + while true; do + sleep 5 + current_size=$(stat -c %s "${coredump_file}") + if [[ ${initial_size} -eq ${current_size} ]]; then + break + else + initial_size=${current_size} + fi + done +} + archive_doris_coredump() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi archive_name="$1" + COREDUMP_SIZE_THRESHOLD="${COREDUMP_SIZE_THRESHOLD:-85899345920}" # if coredump size over 80G, do not archive" if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi - be_pid="$(cat "${DORIS_HOME}"/be/bin/be.pid)" - if [[ -z "${be_id}" ]]; then echo "ERROR: can not find be id from ${DORIS_HOME}/be/bin/be.pid" && return 1; fi - if corename=$(find /var/lib/apport/coredump/ -type f -name "core.*${be_pid}.*"); then - initial_size=$(stat -c %s "${corename}") - while true; do - sleep 2 - current_size=$(stat -c %s "${corename}") - if [[ ${initial_size} -eq ${current_size} ]]; then - break - else - initial_size=${current_size} + local archive_dir="${archive_name%.tar.gz}" + rm -rf "${DORIS_HOME:?}/${archive_dir}" + mkdir -p "${DORIS_HOME}/${archive_dir}" + declare -A pids + pids['be']="$(cat "${DORIS_HOME}"/be/bin/be.pid)" + pids['ms']="$(cat "${DORIS_HOME}"/ms/bin/doris_cloud.pid)" + pids['recycler']="$(cat "${DORIS_HOME}"/recycler/bin/doris_cloud.pid)" + for p in "${!pids[@]}"; do + pid="${pids[${p}]}" + if [[ -z "${pid}" ]]; then continue; fi + if coredump_file=$(find /var/lib/apport/coredump/ -type f -name "core.*${pid}.*") && + wait_coredump_file_ready "${coredump_file}"; then + file_size=$(stat -c %s "${coredump_file}") + if ((file_size <= COREDUMP_SIZE_THRESHOLD)); then + mkdir -p "${DORIS_HOME}/${archive_dir}/${p}" + if [[ "${p}" == "be" ]]; then + mv "${DORIS_HOME}"/be/lib/doris_be "${DORIS_HOME}/${archive_dir}/${p}" + elif [[ "${p}" == "ms" ]]; then + mv "${DORIS_HOME}"/ms/lib/doris_cloud "${DORIS_HOME}/${archive_dir}/${p}" + elif [[ "${p}" == "recycler" ]]; then + mv "${DORIS_HOME}"/recycler/lib/doris_cloud "${DORIS_HOME}/${archive_dir}/${p}" + fi + mv "${coredump_file}" "${DORIS_HOME}/${archive_dir}/${p}" fi - done - file_size=$(stat -c %s "${corename}") - if ((file_size > 85899345920)); then - echo "coredump size ${file_size} over 80G, not upload" - return 1 - else - #压缩core文件 - mv "${corename}" "${DORIS_HOME}"/be/lib/ - cd "${DORIS_HOME}"/be/lib/ || return 1 - tar -I pigz -cf core.tar.gz be/lib/doris_be "be/lib/$(basename "${corename}")" >/dev/null - echo "$(pwd)/core.tar.gz" fi + done + + if tar -I pigz \ + --directory "${DORIS_HOME}" \ + -cf "${DORIS_HOME}/${archive_name}" \ + "${archive_dir}"; then + echo "${DORIS_HOME}/${archive_name}" + else + return 1 fi } diff --git a/regression-test/pipeline/common/oss-utils.sh b/regression-test/pipeline/common/oss-utils.sh index 465d1085e95b0f..4ea966982afc06 100644 --- a/regression-test/pipeline/common/oss-utils.sh +++ b/regression-test/pipeline/common/oss-utils.sh @@ -18,22 +18,22 @@ function install_ossutil() { if command -v ossutil >/dev/null; then return 0; fi - if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then - echo "ERROR: env OSS_accessKeyID or OSS_accessKeySecret not set." + if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then + echo "ERROR: env oss_ak or oss_sk not set." return 1 fi curl https://gosspublic.alicdn.com/ossutil/install.sh | sudo bash echo "[Credentials] language=EN endpoint=oss-cn-hongkong-internal.aliyuncs.com -accessKeyID=${OSS_accessKeyID:-} -accessKeySecret=${OSS_accessKeySecret:-} +accessKeyID=${oss_ak:-} +accessKeySecret=${oss_sk:-} " >~/.ossutilconfig } function check_oss_file_exist() { - if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then - echo "ERROR: env OSS_accessKeyID and OSS_accessKeySecret not set" + if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then + echo "ERROR: env oss_ak and oss_sk not set" return 1 fi # Check if the file exists. @@ -42,8 +42,8 @@ function check_oss_file_exist() { OSS_DIR="${OSS_DIR:-"oss://opensource-pipeline/compile-release"}" install_ossutil if ossutil stat \ - -i "${OSS_accessKeyID}" \ - -k "${OSS_accessKeySecret}" \ + -i "${oss_ak}" \ + -k "${oss_sk}" \ "${OSS_DIR}/${file_name}"; then echo "INFO: ${file_name} file exists." && return 0 else @@ -58,8 +58,8 @@ function download_oss_file() { OSS_DIR="${OSS_DIR:-"oss://opensource-pipeline/compile-release"}" install_ossutil if ossutil cp -f \ - -i "${OSS_accessKeyID}" \ - -k "${OSS_accessKeySecret}" \ + -i "${oss_ak}" \ + -k "${oss_sk}" \ "${OSS_DIR}/${file_name}" \ "${file_name}"; then echo "INFO: download ${file_name} success" && return 0 @@ -69,8 +69,8 @@ function download_oss_file() { } function upload_file_to_oss() { - if [[ -z ${OSS_accessKeyID} || -z ${OSS_accessKeySecret} ]]; then - echo "ERROR: env OSS_accessKeyID and OSS_accessKeySecret not set" + if [[ -z ${oss_ak} || -z ${oss_sk} ]]; then + echo "ERROR: env oss_ak and oss_sk not set" return 1 fi if [[ ! -f "$1" ]] || [[ "$1" != "/"* ]]; then @@ -87,8 +87,8 @@ function upload_file_to_oss() { install_ossutil cd "${dir_name}" || return 1 if ossutil cp -f \ - -i "${OSS_accessKeyID}" \ - -k "${OSS_accessKeySecret}" \ + -i "${oss_ak}" \ + -k "${oss_sk}" \ "${file_name}" \ "${OSS_DIR}/${file_name}"; then if ! check_oss_file_exist "${file_name}"; then return 1; fi