Add Hive full-flow benchmark helpers

docker/thirdparties/benchmark-hive-fullflow.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${ROOT}/.." && pwd)"
+BENCH_DIR="${BENCH_DIR:-${REPO_ROOT}/output/hive-fullflow-bench}"
+mkdir -p "${BENCH_DIR}"
+
+run_case() {
+    local label="$1"
+    shift
+    local log_file="${BENCH_DIR}/${label}.log"
+    local start_ts end_ts elapsed
+
+    echo "INFO: running ${label}"
+    start_ts="$(date +%s)"
+    if "$@" >"${log_file}" 2>&1; then
+        end_ts="$(date +%s)"
+        elapsed="$((end_ts - start_ts))"
+        echo "${label},success,${elapsed},${log_file}"
+        return 0
+    fi
+
+    end_ts="$(date +%s)"
+    elapsed="$((end_ts - start_ts))"
+    echo "${label},failure,${elapsed},${log_file}"
+    return 1
+}
+
+main() {
+    local overall_rc=0
+    {
+        echo "label,status,seconds,log_file"
+        run_case \
+            "current_hive3_fullflow" \
+            env \
+                POC_WORKDIR="${REPO_ROOT}/output/hive3-current-poc" \
+                POC_TMPDIR="${REPO_ROOT}/output/hive3-current-poc/tmp" \
+                HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-180}" \
+                bash "${ROOT}/docker-compose/hive/run-hive3-current-poc.sh" || overall_rc=1
+        run_case \
+            "tez_hive3_fullflow" \
+            env \
+                POC_WORKDIR="${REPO_ROOT}/output/hive3-tez-poc" \
+                POC_TMPDIR="${REPO_ROOT}/output/hive3-tez-poc/tmp" \
+                SERVICE_TIMEOUT_SECONDS="${SERVICE_TIMEOUT_SECONDS:-180}" \
+                HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-240}" \
+                bash "${ROOT}/docker-compose/kerberos/run-hive3-tez-poc.sh" || overall_rc=1
+    } | tee "${BENCH_DIR}/summary.csv"
+
+    exit "${overall_rc}"
+}
+
+main "$@"

docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl

@@ -20,6 +20,7 @@ HIVE_SITE_CONF_hive_metastore_event_db_notification_api_auth=false
 HIVE_SITE_CONF_hive_metastore_dml_events=true
 HIVE_SITE_CONF_hive_metastore_transactional_event_listeners=org.apache.hive.hcatalog.listener.DbNotificationListener
 HIVE_SITE_CONF_hive_stats_column_autogather=false
+HIVE_SITE_CONF_hive_execution_engine=tez
 HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
 HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
 HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
@@ -48,4 +49,3 @@ HIVE_SITE_CONF_fs_gs_auth_service_account_private_key_id=${GCSAccountPrivateKeyI
 HIVE_SITE_CONF_fs_gs_auth_service_account_private_key=${GCSAccountPrivateKey}
 HIVE_SITE_CONF_fs_gs_proxy_address=${GCSProxyAddress}
 enablePaimonHms=${enablePaimonHms}
-

docker/thirdparties/docker-compose/hive/run-hive3-current-poc.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${ROOT}/../../../.." && pwd)"
+WORKDIR="${POC_WORKDIR:-${REPO_ROOT}/output/hive3-current-poc}"
+TMPDIR="${POC_TMPDIR:-${WORKDIR}/tmp}"
+POC_CONTAINER_UID="${POC_CONTAINER_UID:-hive3curpoc}"
+POC_KEEP_UP="${POC_KEEP_UP:-0}"
+HS_PORT="${POC_HS_PORT:-43000}"
+HMS_PORT="${POC_HMS_PORT:-49383}"
+PG_PORT="${POC_PG_PORT:-45732}"
+FS_PORT="${POC_FS_PORT:-48320}"
+LOAD_PARALLEL="${POC_LOAD_PARALLEL:-1}"
+HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-180}"
+IP_HOST="${POC_IP_HOST:-127.0.0.1}"
+
+COMPOSE_DIR="${WORKDIR}/hive"
+COMPOSE_FILE="${COMPOSE_DIR}/hive-3x.yaml"
+ENV_FILE="${COMPOSE_DIR}/hadoop-hive-3x.env"
+SERVER_CONTAINER="${POC_CONTAINER_UID}hive3-server"
+
+cleanup() {
+    if [[ "${POC_KEEP_UP}" == "1" ]]; then
+        echo "INFO: keep current Hive POC environment at ${WORKDIR}"
+        return
+    fi
+
+    if [[ -f "${COMPOSE_FILE}" ]]; then
+        docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" down >/dev/null 2>&1 || true
+    fi
+    rm -rf "${WORKDIR}"
+}
+
+find_beeline() {
+    docker exec "${SERVER_CONTAINER}" bash -lc '
+        if command -v beeline >/dev/null 2>&1; then
+            command -v beeline
+        elif [[ -x /opt/hive/bin/beeline ]]; then
+            echo /opt/hive/bin/beeline
+        else
+            exit 1
+        fi
+    '
+}
+
+probe_beeline() {
+    local beeline_bin="$1"
+    docker exec "${SERVER_CONTAINER}" bash -lc "
+        set -euo pipefail
+        ${beeline_bin} -u 'jdbc:hive2://127.0.0.1:${HS_PORT}/default' -n root --showHeader=false --outputformat=tsv2 -e 'show databases;'
+    "
+}
+
+wait_until_ready() {
+    local beeline_bin waited
+    beeline_bin="$(find_beeline)"
+    waited=0
+    while (( waited < HEALTH_TIMEOUT_SECONDS )); do
+        if probe_beeline "${beeline_bin}" >/dev/null 2>&1; then
+            echo "INFO: current HiveServer2 is ready on ${HS_PORT}"
+            return 0
+        fi
+        if (( waited % 30 == 0 )); then
+            echo "INFO: waiting for current HiveServer2 to become ready"
+        fi
+        sleep 5
+        waited=$((waited + 5))
+    done
+
+    echo "ERROR: timed out waiting for current HiveServer2"
+    docker ps -a --filter "name=${SERVER_CONTAINER}" --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' || true
+    docker logs --tail 200 "${SERVER_CONTAINER}" || true
+    return 1
+}
+
+render_temp_compose() {
+    rm -rf "${WORKDIR}"
+    mkdir -p "${WORKDIR}"
+    mkdir -p "${TMPDIR}"
+    export TMPDIR
+    cp -r "${ROOT}" "${WORKDIR}/"
+
+    export CONTAINER_UID="${POC_CONTAINER_UID}"
+    export NEED_LOAD_DATA=0
+    export LOAD_PARALLEL
+    export IP_HOST
+    export FS_PORT HMS_PORT HS_PORT PG_PORT
+
+    pushd "${COMPOSE_DIR}" >/dev/null
+    envsubst < hadoop-hive.env.tpl > hadoop-hive-3x.env
+    envsubst < hadoop-hive-3x.env.tpl >> hadoop-hive-3x.env
+    envsubst < hive-3x.yaml.tpl > hive-3x.yaml
+    popd >/dev/null
+}
+
+run_poc_queries() {
+    local beeline_bin
+    beeline_bin="$(find_beeline)"
+    docker exec "${SERVER_CONTAINER}" bash -lc "
+        set -euo pipefail
+        ${beeline_bin} -u 'jdbc:hive2://127.0.0.1:${HS_PORT}/default' -n root --showHeader=false --outputformat=tsv2 -e \"
+            drop table if exists hive3_cur_poc_orc;
+            create table hive3_cur_poc_orc(id int) stored as orc;
+            insert into hive3_cur_poc_orc values (1),(2),(3);
+            select count(*) from hive3_cur_poc_orc;
+        \"
+    "
+}
+
+main() {
+    trap cleanup EXIT
+
+    echo "INFO: rendering temporary current Hive3 POC under ${WORKDIR}"
+    render_temp_compose
+
+    echo "INFO: starting current Hive3 compose"
+    docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" down >/dev/null 2>&1 || true
+    docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d --wait
+
+    wait_until_ready
+    run_poc_queries
+
+    echo "INFO: current Hive3 POC finished successfully"
+}
+
+main "$@"