Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

KYLIN-4855 kylin.metrics.prefix bug fix #1536

Merged
merged 2 commits into from
Feb 22, 2021
Merged

KYLIN-4855 kylin.metrics.prefix bug fix #1536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c880691
KYLIN-4855 kylin.metrics.prefix bug fix
wolfboys Jan 4, 2021
be98fd8
KYLIN-4855 kylin.metrics.prefix bug fix
wolfboys Jan 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions build/bin/build-incremental-cube.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh

if [ ! $1 ]; then
echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
exit 1
echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
exit 1
fi

tomcat_root=${dir}/../tomcat
Expand All @@ -41,4 +41,5 @@ END=$((END_TIME - END_TIME%INTERVAL))
ID="$END"
echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log
echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log"
curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
curl -X PUT -H "Authorization: Basic %Auth%" -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &

212 changes: 130 additions & 82 deletions build/bin/system-cube.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh

build_incremental_cube="${KYLIN_HOME}/bin/build-incremental-cube.sh"

function printHelp {
echo "usage: system-cube.sh setup"
echo " system-cube.sh build [INTERVAL:600000] [DELAY:0]"
Expand All @@ -28,6 +30,33 @@ function printHelp {
exit 1
}

function authorization {
authed=$(grep %Auth% "$build_incremental_cube"|wc -l)
if [ $authed -eq 1 ]
then
read -p $'Please Enter the ADMIN\'password or \'N\' to exit: ' pwd
if [ $pwd == "N" ]
then
return 1
else
base64_auth=$(echo -n "ADMIN:$pwd"|base64)
tomcat_root=${dir}/../tomcat
kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2`
http_code=$(curl -I -m 10 -o /dev/null -s -w %{http_code} -X POST -H "Authorization: Basic $base64_auth" -H 'Content-Type: application/json' http://${kylin_rest_address}/kylin/api/user/authentication)
if [ $http_code -eq 200 ]
then
sed -i "s/%Auth%/${base64_auth}/g" $build_incremental_cube
return 0
else
echo `setColor 31 "Unauthorized,password error."`
authorization
fi
fi
else
return 0
fi
}

if [[ "$@" == *"help"* ]]
then
printHelp
Expand All @@ -38,103 +67,122 @@ OUTPUT_FORDER=$KYLIN_HOME/system_cube
KYLIN_ENV=`grep "^kylin.env=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
KYLIN_ENV=${KYLIN_ENV:-"QA"}

SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_${KYLIN_ENV}"
SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
SC_NAME_4="KYLIN_HIVE_METRICS_JOB_${KYLIN_ENV}"
SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"
KYLIN_METRICS_PREFIX=`grep "^kylin.metrics.prefix=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
KYLIN_METRICS_PREFIX=${KYLIN_METRICS_PREFIX:-"KYLIN"}
KYLIN_METRICS_PREFIX=`echo "$KYLIN_METRICS_PREFIX"| tr '[a-z]' '[A-Z]'`

SC_NAME_1="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_${KYLIN_ENV}"
SC_NAME_2="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
SC_NAME_3="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
SC_NAME_4="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_${KYLIN_ENV}"
SC_NAME_5="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"

if [ "$1" == "build" ]
then
if [ -d "${OUTPUT_FORDER}" ]
then
BUILD_INTERVAL=${2:-"600000"}
BUILD_DELAY=${3:-"0"}

echo "build system cubes, build_interval:${BUILD_INTERVAL}, build_delay:${BUILD_DELAY}"

sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh $build_incremental_cube ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh $build_incremental_cube ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh $build_incremental_cube ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh $build_incremental_cube ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh $build_incremental_cube ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
else
echo "Please setup system cube first."
exit 1
echo "Please setup system cube first."
exit 1
fi
elif [ "$1" == "setup" ]
then
#creat SCSinkTools.json
SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json

echo "setup system cubes"

rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER

cat <<-EOF > ${SINK_TOOLS_FILE}
[
{
"sink": "hive",
"storage_type": 2,
"cube_desc_override_properties": {
"kylin.cube.algorithm": "INMEM",
"kylin.cube.max-building-segments": "1"
}
}
]
EOF
$KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
-inputConfig ${SINK_TOOLS_FILE} \
-output ${OUTPUT_FORDER}

hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`

# Get Database, default is KYLIN
system_database="KYLIN"

# 'create database' failed will not exit when donot have permission to create database;
sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql

if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database

hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
if [ -z ${hive2_url} ]; then
hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
fi

beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
fi

$KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}

#refresh signature
$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
#creat SCSinkTools.json
SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json

echo "setup system cubes"

rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER

cat <<-EOF > ${SINK_TOOLS_FILE}
[
{
"sink": "hive",
"storage_type": 2,
"cube_desc_override_properties": {
"kylin.cube.algorithm": "INMEM",
"kylin.cube.max-building-segments": "1"
}
}
]
EOF

$KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
-inputConfig ${SINK_TOOLS_FILE} \
-output ${OUTPUT_FORDER}

hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`

# Get Database, default is KYLIN
system_database="KYLIN"

# 'create database' failed will not exit when donot have permission to create database;
sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql

if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database

hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
if [ -z ${hive2_url} ]; then
hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
fi

beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; }
fi

$KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}

#refresh signature
$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}

elif [ "$1" == "cron" ]
then
#add a crontab job
echo "add to a crontab job"

CRONTAB_FILE=$KYLIN_HOME/crontabJob
crontab -l >> ${CRONTAB_FILE}
cat <<-EOF >> ${CRONTAB_FILE}
0 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
20 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
40 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
30 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
50 */12 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
EOF
crontab ${CRONTAB_FILE}
rm ${CRONTAB_FILE}
#check exists
cron_count=$(crontab -l | grep "${KYLIN_METRICS_PREFIX}_HIVE_METRICS" | wc -l)
if [ $cron_count -eq 5 ]
then
echo `setColor 33 "system cube already exists in crontab"`
exit 0
else
#add a crontab job
echo "add to a crontab job"
authorization
if [[ $? == 1 ]]
then
echo "add to a crontab job exit."
exit 0
else
CRONTAB_FILE=$KYLIN_HOME/crontabJob
crontab -l >> ${CRONTAB_FILE}

cat <<-EOF >> ${CRONTAB_FILE}
0 */2 * * * sh $build_incremental_cube ${SC_NAME_1} 3600000 1200000
20 */2 * * * sh $build_incremental_cube ${SC_NAME_2} 3600000 1200000
40 */4 * * * sh $build_incremental_cube ${SC_NAME_3} 3600000 1200000
30 */4 * * * sh $build_incremental_cube ${SC_NAME_4} 3600000 1200000
50 */12 * * * sh $build_incremental_cube ${SC_NAME_5} 3600000 1200000
EOF
crontab ${CRONTAB_FILE}
rm ${CRONTAB_FILE}
echo "add to a crontab job successful."
fi
fi
else
printHelp
fi