Skip to content

Commit

Permalink
KYLIN-3943 Fix some problems in system-cube.sh
Browse files Browse the repository at this point in the history
  • Loading branch information
@Wayne1c @nichunen
Wayne1c authored and nichunen committed Apr 23, 2019
1 parent 261de78 commit cfabb81
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 37 deletions.
15 changes: 7 additions & 8 deletions build/bin/build-incremental-cube.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,21 @@
# limitations under the License.
#

dir=$(dirname ${0})
if [ -z "$KYLIN_HOME" ]; then
export KYLIN_HOME=${dir}/../
fi
echo KYLIN_HOME is set to $KYLIN_HOME
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh

if [ ! $1 ]; then
echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
exit 1
fi

tomcat_root=${dir}/../tomcat
export tomcat_root

CUBE=$1
INTERVAL=${2:-"3600000"}
DELAY=${3:-"0"}
SERVER="localhost"
PORT="7070"

kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2`

CURRENT_TIME_IN_SECOND=`date +%s`
CURRENT_TIME=$((CURRENT_TIME_IN_SECOND * 1000))
Expand All @@ -42,4 +41,4 @@ END=$((END_TIME - END_TIME%INTERVAL))
ID="$END"
echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log
echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log"
curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${SERVER}:${PORT}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
77 changes: 48 additions & 29 deletions build/bin/system-cube.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,46 @@
#


#check kylin home
if [ -z "$KYLIN_HOME" ]
then
echo 'Please make sure KYLIN_HOME has been set'
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh

function printHelp {
echo "usage: system-cube.sh setup"
echo " system-cube.sh build [INTERVAL:600000] [DELAY:0]"
echo " system-cube.sh cron"
echo " system-cube.sh help"
exit 1
else
echo "KYLIN_HOME is set to ${KYLIN_HOME}"
fi
}

if [[ "$@" == *"help"* ]]
then
printHelp
fi

OUTPUT_FORDER=$KYLIN_HOME/system_cube

SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_QA"
SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_QA"
SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_QA"
SC_NAME_4="KYLIN_HIVE_METRICS_JOB_QA"
SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_QA"
KYLIN_ENV=`grep "^kylin.env=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
KYLIN_ENV=${KYLIN_ENV:-"QA"}

SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_${KYLIN_ENV}"
SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
SC_NAME_4="KYLIN_HIVE_METRICS_JOB_${KYLIN_ENV}"
SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"

if [ "$1" == "build" ]
then
if [ -d "${OUTPUT_FORDER}" ]
then
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 600000 0
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 600000 0
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 600000 0
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 600000 0
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 600000 0
BUILD_INTERVAL=${2:-"600000"}
BUILD_DELAY=${3:-"0"}

echo "build system cubes, build_interval:${BUILD_INTERVAL}, build_delay:${BUILD_DELAY}"

sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
else
echo "Please setup system cube first."
exit 1
Expand All @@ -53,7 +66,12 @@ elif [ "$1" == "setup" ]
then
#creat SCSinkTools.json
SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
cat <<- EOF > ${SINK_TOOLS_FILE}

echo "setup system cubes"

rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER

cat <<-EOF > ${SINK_TOOLS_FILE}
[
[
"org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool",
Expand All @@ -80,20 +98,21 @@ then

#refresh signature
$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}

elif [ "$1" == "cron" ]
then
#add a crontab job
echo "add to a crontab job"

CRONTAB_FILE=$KYLIN_HOME/crontabJob
cat <<-EOF > ${CRONTAB_FILE}
0 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
20 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
40 */8 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
30 */8 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
50 */24 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
cat <<-EOF > ${CRONTAB_FILE}
0 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
20 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
40 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
30 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
50 */12 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
EOF
crontab ${CRONTAB_FILE}
rm ${CRONTAB_FILE}
else
echo "usage: system-cube.sh setup"
echo " system-cube.sh build"
exit 1
fi
printHelp
fi

0 comments on commit cfabb81

Please sign in to comment.