KYLIN-4855 kylin.metrics.prefix bug fix (#1536)

* KYLIN-4855 kylin.metrics.prefix bug fix * KYLIN-4855 kylin.metrics.prefix bug fix
apache · Feb 22, 2021 · d1c99d7 · d1c99d7
1 parent 985ff83
commit d1c99d7
Show file tree

Hide file tree

Showing 2 changed files with 134 additions and 85 deletions.
diff --git a/build/bin/build-incremental-cube.sh b/build/bin/build-incremental-cube.sh
@@ -20,8 +20,8 @@
 source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh
 
 if [ ! $1 ]; then
-	echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
-	exit 1
+    echo "usage: build-incremental-cube.sh CUBE INTERVAL DELAY"
+    exit 1
 fi
 
 tomcat_root=${dir}/../tomcat
@@ -41,4 +41,5 @@ END=$((END_TIME - END_TIME%INTERVAL))
 ID="$END"
 echo "Building for ${CUBE}_${ID}" | tee ${KYLIN_HOME}/logs/build_trace.log
 echo "Check the log at ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log"
-curl -X PUT --user ADMIN:KYLIN -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild  > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
+curl -X PUT -H "Authorization: Basic %Auth%" -H "Content-Type: application/json;charset=utf-8" -d "{\"endTime\": ${END}, \"buildType\": \"BUILD\"}" http://${kylin_rest_address}/kylin/api/cubes/${CUBE}/rebuild  > ${KYLIN_HOME}/logs/incremental_cube_${CUBE}_${END}.log 2>&1 &
+
diff --git a/build/bin/system-cube.sh b/build/bin/system-cube.sh
@@ -20,6 +20,8 @@
 
 source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh
 
+build_incremental_cube="${KYLIN_HOME}/bin/build-incremental-cube.sh"
+
 function printHelp {
     echo "usage: system-cube.sh setup"
     echo "       system-cube.sh build [INTERVAL:600000] [DELAY:0]"
@@ -28,6 +30,33 @@ function printHelp {
     exit 1
 }
 
+function authorization {
+    authed=$(grep %Auth% "$build_incremental_cube"|wc -l)
+    if [ $authed -eq 1 ]
+    then
+        read -p $'Please Enter the ADMIN\'password or \'N\' to exit: ' pwd
+        if [ $pwd == "N"  ]
+        then
+            return 1
+        else
+            base64_auth=$(echo -n "ADMIN:$pwd"|base64)
+            tomcat_root=${dir}/../tomcat
+            kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2`
+            http_code=$(curl -I -m 10 -o /dev/null -s -w %{http_code} -X POST -H "Authorization: Basic $base64_auth" -H 'Content-Type: application/json' http://${kylin_rest_address}/kylin/api/user/authentication)
+            if [ $http_code -eq 200  ]
+            then
+                sed -i "s/%Auth%/${base64_auth}/g" $build_incremental_cube
+                return 0
+            else
+                echo `setColor 31 "Unauthorized,password error."`
+                authorization
+            fi
+        fi
+    else
+        return 0
+    fi
+}
+
 if [[ "$@" == *"help"* ]]
 then
     printHelp
@@ -38,103 +67,122 @@ OUTPUT_FORDER=$KYLIN_HOME/system_cube
 KYLIN_ENV=`grep "^kylin.env=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
 KYLIN_ENV=${KYLIN_ENV:-"QA"}
 
-SC_NAME_1="KYLIN_HIVE_METRICS_QUERY_${KYLIN_ENV}"
-SC_NAME_2="KYLIN_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
-SC_NAME_3="KYLIN_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
-SC_NAME_4="KYLIN_HIVE_METRICS_JOB_${KYLIN_ENV}"
-SC_NAME_5="KYLIN_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"
+KYLIN_METRICS_PREFIX=`grep "^kylin.metrics.prefix=" $KYLIN_HOME/conf/kylin.properties | cut -d "=" -f 2`
+KYLIN_METRICS_PREFIX=${KYLIN_METRICS_PREFIX:-"KYLIN"}
+KYLIN_METRICS_PREFIX=`echo "$KYLIN_METRICS_PREFIX"| tr '[a-z]' '[A-Z]'`
+
+SC_NAME_1="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_${KYLIN_ENV}"
+SC_NAME_2="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_CUBE_${KYLIN_ENV}"
+SC_NAME_3="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_QUERY_RPC_${KYLIN_ENV}"
+SC_NAME_4="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_${KYLIN_ENV}"
+SC_NAME_5="${KYLIN_METRICS_PREFIX}_HIVE_METRICS_JOB_EXCEPTION_${KYLIN_ENV}"
 
 if [ "$1" == "build" ]
 then
     if [ -d "${OUTPUT_FORDER}" ]
     then
         BUILD_INTERVAL=${2:-"600000"}
         BUILD_DELAY=${3:-"0"}
-
         echo "build system cubes, build_interval:${BUILD_INTERVAL}, build_delay:${BUILD_DELAY}"
-
-		sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
-		sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
-		sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
-		sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
-		sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
+        sh $build_incremental_cube ${SC_NAME_1} ${BUILD_INTERVAL} ${BUILD_DELAY}
+        sh $build_incremental_cube ${SC_NAME_2} ${BUILD_INTERVAL} ${BUILD_DELAY}
+        sh $build_incremental_cube ${SC_NAME_3} ${BUILD_INTERVAL} ${BUILD_DELAY}
+        sh $build_incremental_cube ${SC_NAME_4} ${BUILD_INTERVAL} ${BUILD_DELAY}
+        sh $build_incremental_cube ${SC_NAME_5} ${BUILD_INTERVAL} ${BUILD_DELAY}
     else
-    	echo "Please setup system cube first."
-		exit 1
+    	  echo "Please setup system cube first."
+		    exit 1
     fi
 elif [ "$1" == "setup" ]
 then
-	#creat SCSinkTools.json
-	SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
-
-	echo "setup system cubes"
-
-	rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER
-
-	cat <<-EOF > ${SINK_TOOLS_FILE}
-	[
-    {
-       "sink": "hive",
-       "storage_type": 2,
-       "cube_desc_override_properties": {
-         "kylin.cube.algorithm": "INMEM",
-         "kylin.cube.max-building-segments": "1"
-       }
-    }
-	]
-	EOF
-  $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
-  -inputConfig ${SINK_TOOLS_FILE} \
-  -output ${OUTPUT_FORDER}
-
-  hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
-
-  # Get Database, default is KYLIN
-  system_database="KYLIN"
-
-  # 'create database' failed will not exit when donot have permission to create database;
-  sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql
-
-  if [ "${hive_client_mode}" == "beeline" ]
-  then
-      beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
-      beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database
-
-      hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
-      if [ -z ${hive2_url} ]; then
-          hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
-          beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
-      else
-          beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
-      fi
-
-      beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql  || { exit 1; }
-  else
-      hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
-      hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql  || { exit 1; }
-  fi
-
-  $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}
-
-  #refresh signature
-  $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
+    #creat SCSinkTools.json
+    SINK_TOOLS_FILE=$KYLIN_HOME/SCSinkTools.json
+
+    echo "setup system cubes"
+
+    rm -rf $SINK_TOOLS_FILE $OUTPUT_FORDER
+
+cat <<-EOF > ${SINK_TOOLS_FILE}
+[
+  {
+     "sink": "hive",
+     "storage_type": 2,
+     "cube_desc_override_properties": {
+       "kylin.cube.algorithm": "INMEM",
+       "kylin.cube.max-building-segments": "1"
+     }
+  }
+]
+EOF
+
+    $KYLIN_HOME/bin/kylin.sh org.apache.kylin.tool.metrics.systemcube.SCCreator \
+    -inputConfig ${SINK_TOOLS_FILE} \
+    -output ${OUTPUT_FORDER}
+
+    hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
+
+    # Get Database, default is KYLIN
+    system_database="KYLIN"
+
+    # 'create database' failed will not exit when donot have permission to create database;
+    sed -i -e 's/CREATE DATABASE /-- CREATE DATABASE /g' ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql
+
+    if [ "${hive_client_mode}" == "beeline" ]
+    then
+        beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
+        beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$system_database
+
+        hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
+        if [ -z ${hive2_url} ]; then
+            hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
+            beeline_params=${beeline_params/${hive2_url}/${hive2_url}/${system_database}}
+        else
+            beeline_params=${beeline_params/${hive2_url}/${hive2_url}${system_database}}
+        fi
+
+        beeline ${beeline_params} -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql  || { exit 1; }
+    else
+        hive -e "CREATE DATABASE IF NOT EXISTS "$system_database
+        hive --database $system_database -f ${OUTPUT_FORDER}/create_hive_tables_for_system_cubes.sql  || { exit 1; }
+    fi
+
+    $KYLIN_HOME/bin/metastore.sh restore ${OUTPUT_FORDER}
+
+    #refresh signature
+    $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher ${SC_NAME_1},${SC_NAME_2},${SC_NAME_3},${SC_NAME_4},${SC_NAME_5}
 
 elif [ "$1" == "cron" ]
 then
-    #add a crontab job
-    echo "add to a crontab job"
-
-    CRONTAB_FILE=$KYLIN_HOME/crontabJob
-    	crontab -l >> ${CRONTAB_FILE}
-	cat <<-EOF >> ${CRONTAB_FILE}
-    0 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_1} 3600000 1200000
-    20 */2 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_2} 3600000 1200000
-    40 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_3} 3600000 1200000
-    30 */4 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_4} 3600000 1200000
-    50 */12 * * * sh ${KYLIN_HOME}/bin/build-incremental-cube.sh ${SC_NAME_5} 3600000 1200000
-	EOF
-    crontab ${CRONTAB_FILE}
-    rm ${CRONTAB_FILE}
+    #check exists
+    cron_count=$(crontab -l | grep "${KYLIN_METRICS_PREFIX}_HIVE_METRICS" | wc -l)
+    if [ $cron_count -eq 5 ]
+    then
+        echo `setColor 33 "system cube already exists in crontab"`
+        exit 0
+    else
+        #add a crontab job
+        echo "add to a crontab job"
+        authorization
+        if [[ $? == 1 ]]
+        then
+            echo "add to a crontab job exit."
+            exit 0
+        else
+            CRONTAB_FILE=$KYLIN_HOME/crontabJob
+            crontab -l >> ${CRONTAB_FILE}
+
+cat <<-EOF >> ${CRONTAB_FILE}
+0 */2 * * * sh $build_incremental_cube ${SC_NAME_1} 3600000 1200000
+20 */2 * * * sh $build_incremental_cube ${SC_NAME_2} 3600000 1200000
+40 */4 * * * sh $build_incremental_cube ${SC_NAME_3} 3600000 1200000
+30 */4 * * * sh $build_incremental_cube ${SC_NAME_4} 3600000 1200000
+50 */12 * * * sh $build_incremental_cube ${SC_NAME_5} 3600000 1200000
+EOF
+            crontab ${CRONTAB_FILE}
+            rm ${CRONTAB_FILE}
+            echo "add to a crontab job successful."
+        fi
+    fi
 else
     printHelp
 fi