diff --git a/build/bin/build-incremental-cube.sh b/build/bin/build-incremental-cube.sh index 04eee35904b..28704721438 100644 --- a/build/bin/build-incremental-cube.sh +++ b/build/bin/build-incremental-cube.sh @@ -20,8 +20,8 @@ source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh if [ ! $1 ]; then - echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY" - exit 1 + echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY" + exit 1 fi tomcat_root=${dir}/../tomcat @@ -41,4 +41,5 @@ END=$((END_TIME - END_TIME%INTERVAL)) ID="$END" echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log" -curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 & +curl -X PUT -H "Authorization: Basic %Auth%" -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 & + diff --git a/build/bin/system-cube.sh b/build/bin/system-cube.sh index b5d01b9721d..dbab6a9f79b 100644 --- a/build/bin/system-cube.sh +++ b/build/bin/system-cube.sh @@ -20,6 +20,8 @@ source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh +build_incremental_cube="${KYLIN_HOME}/bin/build-incremental-cube.sh" + function printHelp { echo "usage: system-cube.sh setup" echo " system-cube.sh build [INTERVAL:600000] [DELAY:0]" @@ -28,6 +30,33 @@ function printHelp { exit 1 } +function authorization { + authed=$(grep %Auth% "$build_incremental_cube"|wc -l) + if [ $authed -eq 1 ] + then + read -p $'Please Enter the ADMIN\'password or \'N\' to exit: ' pwd + if [ $pwd == "N" ] + then + return 1 + else + base64_auth=$(echo -n "ADMIN:$pwd"|base64) + tomcat_root=${dir}/../tomcat + kylin_rest_address=`hostname -f`":"`grep " ${SINK_TOOLS_FILE} - [ - { - "sink": "hive", - "storage_type": 2, - "cube_desc_override_properties": { - "kylin.cube.algorithm": "INMEM", - "kylin.cube.max-building-segments": "1" - } - } - ] - EOF - $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \ - -inputConfig ${SINK_TOOLS_FILE} \ - -output ${OUTPUT_FORDER} - - hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client` - - # Get Database, default is KYLIN - system_database="KYLIN" - - # 'create database' failed will not exit when donot have permission to create database; - sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql - - if [ "${hive_client_mode}" == "beeline" ] - then - beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params` - beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database - - hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'` - if [ -z ${hive2_url} ]; then - hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'` - beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}} - else - beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}} - fi - - beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; } - else - hive -e "CREATE DATABASE IF NOT EXISTS "$system_database - hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; } - fi - - $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER} - - #refresh signature - $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5} + #creat SCSinkTools.json + SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json + + echo "setup system cubes" + + rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER + +cat <<-EOF > ${SINK_TOOLS_FILE} +[ + { + "sink": "hive", + "storage_type": 2, + "cube_desc_override_properties": { + "kylin.cube.algorithm": "INMEM", + "kylin.cube.max-building-segments": "1" + } + } +] +EOF + + $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \ + -inputConfig ${SINK_TOOLS_FILE} \ + -output ${OUTPUT_FORDER} + + hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client` + + # Get Database, default is KYLIN + system_database="KYLIN" + + # 'create database' failed will not exit when donot have permission to create database; + sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql + + if [ "${hive_client_mode}" == "beeline" ] + then + beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params` + beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database + + hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'` + if [ -z ${hive2_url} ]; then + hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'` + beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}} + else + beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}} + fi + + beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; } + else + hive -e "CREATE DATABASE IF NOT EXISTS "$system_database + hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql || { exit 1; } + fi + + $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER} + + #refresh signature + $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5} elif [ "$1" == "cron" ] then - #add a crontab job - echo "add to a crontab job" - - CRONTAB_FILE=$KYLIN_HOME/crontabJob - crontab -l >> ${CRONTAB_FILE} - cat <<-EOF >> ${CRONTAB_FILE} - 0 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000 - 20 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000 - 40 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000 - 30 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000 - 50 */12 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000 - EOF - crontab ${CRONTAB_FILE} - rm ${CRONTAB_FILE} + #check exists + cron_count=$(crontab -l | grep "${KYLIN_METRICS_PREFIX}_HIVE_METRICS" | wc -l) + if [ $cron_count -eq 5 ] + then + echo `setColor 33 "system cube already exists in crontab"` + exit 0 + else + #add a crontab job + echo "add to a crontab job" + authorization + if [[ $? == 1 ]] + then + echo "add to a crontab job exit." + exit 0 + else + CRONTAB_FILE=$KYLIN_HOME/crontabJob + crontab -l >> ${CRONTAB_FILE} + +cat <<-EOF >> ${CRONTAB_FILE} +0 */2 * * * sh $build_incremental_cube ${SC_NAME_1} 3600000 1200000 +20 */2 * * * sh $build_incremental_cube ${SC_NAME_2} 3600000 1200000 +40 */4 * * * sh $build_incremental_cube ${SC_NAME_3} 3600000 1200000 +30 */4 * * * sh $build_incremental_cube ${SC_NAME_4} 3600000 1200000 +50 */12 * * * sh $build_incremental_cube ${SC_NAME_5} 3600000 1200000 +EOF + crontab ${CRONTAB_FILE} + rm ${CRONTAB_FILE} + echo "add to a crontab job successful." + fi + fi else printHelp fi