From e30007821cce397864b140fd501add670c769f97 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Thu, 5 Aug 2021 15:02:29 +0800 Subject: [PATCH 1/5] add rewrite manifest opt --- start.sh | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/start.sh b/start.sh index 849d18450fc..bd58fd2d036 100644 --- a/start.sh +++ b/start.sh @@ -1,15 +1,98 @@ #!/bin/bash APP=$1 +MANIFEST_OPT=$2 +ALL_OPT=$* +NEED_REBUILD=0 + +if [[ $1 == '--rewrite--manifest' ]] || [[ $1 == '--r' ]] ; then + APP='' + NEED_REBUILD=1 + elif [[ $1 == '--rewrite--manifest' ]] || [[ $1 == '--r' ]] ; then + NEED_REBUILD=1 +fi + +rebuildManifest() { + if [[ $NEED_REBUILD == 1 ]] ; then + buildManifest + fi +} + + +buildManifest() { + ARCHIVE_JAR='ArchiveManifest.jar' + java -jar $ARCHIVE_JAR $ALL_OPT + ret=$? + if [[ $ret == 0 ]] ; then + echo 'rebuild manifest success' + else + echo 'rebuild manifest fail, log in logs/archive.log' + fi + return ret +} + +checkMemory() { + ALLOW_MEMORY=16000000 + ALLOW_MAX_MEMORY=32000000 + MAX_MATESPACE_SIZE=' -XX:MaxMetaspaceSize=512m ' + total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` + # total < ALLOW_MEN + if [ $total -lt $ALLOW_MEMORY ] ; then + echo "Direct memory must be greater than $ALLOW_MEMORY!, current memory: $total!!" + exit + fi + if [[ $total -gt $ALLOW_MEMORY ]] && [[ $total -lt $ALLOW_MAX_MEMORY ]] ; then +echo 1 $total + MAX_NEW_SIZE=' -XX:NewSize=3072m -XX:MaxNewSize=3072m ' + MEM_OPT="$MAX_MATESPACE_SIZE $MAX_NEW_SIZE" + + elif [[ $total -gt $ALLOW_MEMORY ]] ; then +echo 2 $total + NEW_RATIO=' -XX:NewSize=6144m -XX:MaxNewSize=6144m ' + MEM_OPT="$MAX_MATESPACE_SIZE $NEW_RATIO" + fi +} + APP=${APP:-"FullNode"} START_OPT=`echo ${@:2}` JAR_NAME="$APP.jar" MAX_STOP_TIME=60 +MEM_OPT='' checkpid() { pid=`ps -ef | grep $JAR_NAME |grep -v grep | awk '{print $2}'` return $pid } +checkPath(){ + path='output-directory/database' + flag=1 + for p in ${ALL_OPT} + do + if [[ $flag == 0 ]] ; then + path=`echo $p` + break + fi + if [[ $p == '-d' || $p == '--database-directory' ]] ; then + path='' + flag=0 + fi + done + + if [[ -z "${path}" ]]; then + echo '-d /path or --database-directory /path' + return 1 + fi + + if [[ -d ${path} ]]; then + return 0 + else + echo $path 'not exist' + return 1 + fi +} + + + stopService() { count=1 while [ $count -le $MAX_STOP_TIME ]; do @@ -32,15 +115,24 @@ stopService() { startService() { echo `date` >> start.log total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` - xmx=`echo "$total/1024/1024*0.8" | bc |awk -F. '{print $1"g"}'` + xmx=`echo "$total/1024/1024*0.6" | bc |awk -F. '{print $1"g"}'` + directmem=`echo "$total/1024/1024*0.1" | bc |awk -F. '{print $1"g"}'` logtime=`date +%Y-%m-%d_%H-%M-%S` - nohup java -Xmx$xmx -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -Xloggc:./gc.log\ - -XX:+PrintGCDateStamps -XX:+CMSParallelRemarkEnabled -XX:ReservedCodeCacheSize=256m\ - -XX:+CMSScavengeBeforeRemark -jar $JAR_NAME $START_OPT -c config.conf >> start.log 2>&1 & - + export LD_PRELOAD="/usr/lib64/libtcmalloc.so" + nohup java -Xms$xmx -Xmx$xmx -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -Xloggc:./gc.log\ + -XX:+PrintGCDateStamps -XX:+CMSParallelRemarkEnabled -XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing\ + $MEM_OPT -XX:MaxDirectMemorySize=$directmem -XX:+HeapDumpOnOutOfMemoryError -jar $JAR_NAME $START_OPT -c config.conf >> start.log 2>&1 & pid=`ps -ef |grep $JAR_NAME |grep -v grep |awk '{print $2}'` echo "start java-tron with pid $pid on $HOSTNAME" } stopService +checkPath +if [[ 0 == $? ]] ; then + rebuildManifest +else + exit -1 +fi +sleep 5 +checkMemory startService From 2897dd247329f70cb7f5fba0c6b65827d1b112c5 Mon Sep 17 00:00:00 2001 From: liukai Date: Fri, 6 Aug 2021 16:10:08 +0800 Subject: [PATCH 2/5] add input direct param --- start.sh | 82 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/start.sh b/start.sh index bd58fd2d036..9cc5546a876 100644 --- a/start.sh +++ b/start.sh @@ -1,13 +1,14 @@ #!/bin/bash APP=$1 -MANIFEST_OPT=$2 ALL_OPT=$* NEED_REBUILD=0 +ALL_ARR_OPT=($@) +ALL_OPT_NUM=$# if [[ $1 == '--rewrite--manifest' ]] || [[ $1 == '--r' ]] ; then APP='' NEED_REBUILD=1 - elif [[ $1 == '--rewrite--manifest' ]] || [[ $1 == '--r' ]] ; then + elif [[ $2 == '--rewrite--manifest' ]] || [[ $2 == '--r' ]] ; then NEED_REBUILD=1 fi @@ -17,10 +18,19 @@ rebuildManifest() { fi } - buildManifest() { ARCHIVE_JAR='ArchiveManifest.jar' - java -jar $ARCHIVE_JAR $ALL_OPT + if [[ -f $archive_jar ]] ; then + java -jar $archive_jar $all_opt + else + echo 'download archivemanifest.jar' + download=`wget https://github.com/tronprotocol/java-tron/releases/download/greatvoyage-v4.3.0/archivemanifest.jar` + if [[ $download == 0 ]] ; then + echo 'download success, rebuild manifest' + java -jar $archive_jar $all_opt + fi + fi + ret=$? if [[ $ret == 0 ]] ; then echo 'rebuild manifest success' @@ -30,25 +40,49 @@ buildManifest() { return ret } -checkMemory() { - ALLOW_MEMORY=16000000 - ALLOW_MAX_MEMORY=32000000 - MAX_MATESPACE_SIZE=' -XX:MaxMetaspaceSize=512m ' - total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` - # total < ALLOW_MEN - if [ $total -lt $ALLOW_MEMORY ] ; then - echo "Direct memory must be greater than $ALLOW_MEMORY!, current memory: $total!!" - exit - fi - if [[ $total -gt $ALLOW_MEMORY ]] && [[ $total -lt $ALLOW_MAX_MEMORY ]] ; then -echo 1 $total - MAX_NEW_SIZE=' -XX:NewSize=3072m -XX:MaxNewSize=3072m ' - MEM_OPT="$MAX_MATESPACE_SIZE $MAX_NEW_SIZE" - - elif [[ $total -gt $ALLOW_MEMORY ]] ; then -echo 2 $total - NEW_RATIO=' -XX:NewSize=6144m -XX:MaxNewSize=6144m ' - MEM_OPT="$MAX_MATESPACE_SIZE $NEW_RATIO" +checkmemory() { + allow_memory=8000000 + allow_max_memory=48000000 + max_matespace_size=' -xx:maxmetaspacesize=512m ' + total=`cat /proc/meminfo |grep memtotal |awk -f ' ' '{print $2}'` + default_memory=true + + position=0 + for param in $ALL_ARR_OPT + do + if [[ $param == '-mem' ]]; then + arr_index=$[position+1] + memory=${ALL_ARR_OPT[position+1]} + echo 'input direct memory:' $memory'MB' + memory=$[memory * 1000] + if [[ $memory =~ ^[0-9]*$ ]] && [[ $memory -gt $allow_memory ]]; then + allow_memory=$memory + default_memory=false + else + echo "direct memory must be greater than1111 $allow_memory!, current memory: $total!!" + fi + fi + position=$[position+1] + done + + if [ $default_memory == true ]; then + # total < allow_mem + if [ $total -lt $allow_memory ] ; then + echo "direct memory must be greater than $allow_memory!, current memory: $total!!" + exit + fi + if [[ $total -gt $allow_memory ]] && [[ $total -lt $allow_max_memory ]] ; then + MAX_NEW_SIZE=' -XX:NewSize=3072m -XX:MaxNewSize=3072m ' + MEM_OPT="$max_matespace_size $max_new_size" + + elif [[ $total -gt $allow_memory ]] ; then + NEW_RATIO=' -XX:NewSize=6144m -XX:MaxNewSize=6144m ' + MEM_OPT="$max_matespace_size $new_ratio" + fi +else + NEW_RATIO=2 + max_matespace_size=$allow_memory / 16 + MEM_OPT="$max_matespace_size $new_ratio" fi } @@ -134,5 +168,5 @@ else exit -1 fi sleep 5 -checkMemory +checkmemory startService From 9d4c18e0b8f8fddd18c60818fb6740fb81fe8ebb Mon Sep 17 00:00:00 2001 From: liukai Date: Mon, 23 Aug 2021 15:33:57 +0800 Subject: [PATCH 3/5] modify -men opt --- start.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/start.sh b/start.sh index 9cc5546a876..45a26814730 100644 --- a/start.sh +++ b/start.sh @@ -44,7 +44,7 @@ checkmemory() { allow_memory=8000000 allow_max_memory=48000000 max_matespace_size=' -xx:maxmetaspacesize=512m ' - total=`cat /proc/meminfo |grep memtotal |awk -f ' ' '{print $2}'` + total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` default_memory=true position=0 @@ -66,8 +66,7 @@ checkmemory() { done if [ $default_memory == true ]; then - # total < allow_mem - if [ $total -lt $allow_memory ] ; then + if [[ $total -lt $allow_memory ]] ; then echo "direct memory must be greater than $allow_memory!, current memory: $total!!" exit fi @@ -81,11 +80,16 @@ checkmemory() { fi else NEW_RATIO=2 - max_matespace_size=$allow_memory / 16 + max_matespace_size=$[allow_memory / 16] MEM_OPT="$max_matespace_size $new_ratio" fi } +if [[ $APP =~ '-' ]]; then + APP='' +fi + + APP=${APP:-"FullNode"} START_OPT=`echo ${@:2}` JAR_NAME="$APP.jar" @@ -125,8 +129,6 @@ checkPath(){ fi } - - stopService() { count=1 while [ $count -le $MAX_STOP_TIME ]; do @@ -160,6 +162,7 @@ startService() { echo "start java-tron with pid $pid on $HOSTNAME" } + stopService checkPath if [[ 0 == $? ]] ; then @@ -169,4 +172,4 @@ else fi sleep 5 checkmemory -startService +startService \ No newline at end of file From b74fb95f4130e3545cc7e058f838b82ad3d6245f Mon Sep 17 00:00:00 2001 From: liukai Date: Thu, 26 Aug 2021 15:37:47 +0800 Subject: [PATCH 4/5] add checkAppName --- start.sh | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/start.sh b/start.sh index 45a26814730..fa3a9c91961 100644 --- a/start.sh +++ b/start.sh @@ -43,7 +43,7 @@ buildManifest() { checkmemory() { allow_memory=8000000 allow_max_memory=48000000 - max_matespace_size=' -xx:maxmetaspacesize=512m ' + max_matespace_size=' -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m ' total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` default_memory=true @@ -59,7 +59,7 @@ checkmemory() { allow_memory=$memory default_memory=false else - echo "direct memory must be greater than1111 $allow_memory!, current memory: $total!!" + echo "direct memory must be greater than $allow_memory!, current memory: $total!!" fi fi position=$[position+1] @@ -71,7 +71,7 @@ checkmemory() { exit fi if [[ $total -gt $allow_memory ]] && [[ $total -lt $allow_max_memory ]] ; then - MAX_NEW_SIZE=' -XX:NewSize=3072m -XX:MaxNewSize=3072m ' + max_new_size=' -XX:NewSize=3072m -XX:MaxNewSize=3072m ' MEM_OPT="$max_matespace_size $max_new_size" elif [[ $total -gt $allow_memory ]] ; then @@ -85,23 +85,23 @@ else fi } -if [[ $APP =~ '-' ]]; then - APP='' -fi - - -APP=${APP:-"FullNode"} -START_OPT=`echo ${@:2}` -JAR_NAME="$APP.jar" -MAX_STOP_TIME=60 -MEM_OPT='' +checkAppName() { + if [[ $APP =~ '-' ]]; then + APP='' + fi + APP=${APP:-"FullNode"} + START_OPT=`echo ${@:2}` + JAR_NAME="$APP.jar" + MAX_STOP_TIME=60 + MEM_OPT='' +} checkpid() { pid=`ps -ef | grep $JAR_NAME |grep -v grep | awk '{print $2}'` return $pid } -checkPath(){ +checkPathAndRebuild(){ path='output-directory/database' flag=1 for p in ${ALL_OPT} @@ -118,18 +118,19 @@ checkPath(){ if [[ -z "${path}" ]]; then echo '-d /path or --database-directory /path' - return 1 + exit -1 fi if [[ -d ${path} ]]; then - return 0 + rebuildManifest else echo $path 'not exist' - return 1 + exit -1 fi } stopService() { + checkAppName count=1 while [ $count -le $MAX_STOP_TIME ]; do checkpid @@ -149,6 +150,8 @@ stopService() { } startService() { + checkmemory + echo `date` >> start.log total=`cat /proc/meminfo |grep MemTotal |awk -F ' ' '{print $2}'` xmx=`echo "$total/1024/1024*0.6" | bc |awk -F. '{print $1"g"}'` @@ -162,14 +165,7 @@ startService() { echo "start java-tron with pid $pid on $HOSTNAME" } - stopService -checkPath -if [[ 0 == $? ]] ; then - rebuildManifest -else - exit -1 -fi +checkPathAndRebuild sleep 5 -checkmemory startService \ No newline at end of file From 9af6acaaf2855d2dbed275714ad78f8c6f2fe889 Mon Sep 17 00:00:00 2001 From: liukai Date: Fri, 27 Aug 2021 12:19:41 +0800 Subject: [PATCH 5/5] add tcmalloc release rate value --- start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/start.sh b/start.sh index fa3a9c91961..8c681ab6640 100644 --- a/start.sh +++ b/start.sh @@ -158,6 +158,7 @@ startService() { directmem=`echo "$total/1024/1024*0.1" | bc |awk -F. '{print $1"g"}'` logtime=`date +%Y-%m-%d_%H-%M-%S` export LD_PRELOAD="/usr/lib64/libtcmalloc.so" + export TCMALLOC_RELEASE_RATE=10 nohup java -Xms$xmx -Xmx$xmx -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -Xloggc:./gc.log\ -XX:+PrintGCDateStamps -XX:+CMSParallelRemarkEnabled -XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing\ $MEM_OPT -XX:MaxDirectMemorySize=$directmem -XX:+HeapDumpOnOutOfMemoryError -jar $JAR_NAME $START_OPT -c config.conf >> start.log 2>&1 &