Optimize the bash script (#29)

### What changes were proposed in this pull request? Optimzie the bash script when deploying. Main changelog 1. Use the `bin/hadoop` to get the hadoop deps. 2. Separate the log file for coordinator and server. ### Why are the changes needed? To be compatible with the other hadoop distribution Including Apache Hadoop, like CDH. ### Does this PR introduce _any_ user-facing change? Yes. ### How was this patch tested? Manual test.
apache · Jul 8, 2022 · ae674a6 · ae674a6
1 parent 4a51ff3
commit ae674a6
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 19 deletions.
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
@@ -32,8 +32,15 @@ cd $COORDINATOR_HOME
 source "${COORDINATOR_HOME}/bin/rss-env.sh"
 source "${COORDINATOR_HOME}/bin/utils.sh"
 
+if [ -z "$HADOOP_HOME" ]; then
+  echo "No env HADOOP_HOME."
+  exit 1
+fi
+
+export JAVA_HOME
+
 HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-HADOOP_DEPENDENCY=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*
+HADOOP_DEPENDENCY=`$HADOOP_HOME/bin/hadoop classpath --glob`
 
 echo "Check process existence"
 is_jvm_process_running $JPS $MAIN_CLASS
@@ -45,11 +52,6 @@ for file in $(ls ${JAR_DIR}/coordinator/*.jar 2>/dev/null); do
   CLASSPATH=$CLASSPATH:$file
 done
 
-if [ -z "$HADOOP_HOME" ]; then
-  echo "No env HADOOP_HOME."
-  exit 1
-fi
-
 if [ -z "$HADOOP_CONF_DIR" ]; then
   echo "No env HADOOP_CONF_DIR."
   exit 1
@@ -71,10 +73,13 @@ JVM_ARGS=" -server \
           -XX:InitiatingHeapOccupancyPercent=45 "
 
 ARGS=""
-if [ -f ./conf/log4j.properties ]; then
-  ARGS="$ARGS -Dlog4j.configuration=file:./conf/log4j.properties"
+
+LOG_CONF_FILE="./conf/log4j.properties"
+LOG_PATH="./logs/coordinator.log"
+if [ -f ${LOG_CONF_FILE} ]; then
+  ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
 else
-  echo "Exit with error: $conf/log4j.properties file doesn't exist."
+  echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1
 fi
 

diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
@@ -30,8 +30,15 @@ cd $SHUFFLE_SERVER_HOME
 source "${SHUFFLE_SERVER_HOME}/bin/rss-env.sh"
 source "${SHUFFLE_SERVER_HOME}/bin/utils.sh"
 
+if [ -z "$HADOOP_HOME" ]; then
+  echo "No env HADOOP_HOME."
+  exit 1
+fi
+
+export JAVA_HOME
+
 HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-HADOOP_DEPENDENCY=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*
+HADOOP_DEPENDENCY=`$HADOOP_HOME/bin/hadoop classpath --glob`
 
 CONF_FILE="./conf/server.conf "
 MAIN_CLASS="org.apache.uniffle.server.ShuffleServer"
@@ -46,11 +53,6 @@ for file in $(ls ${JAR_DIR}/server/*.jar 2>/dev/null); do
   CLASSPATH=$CLASSPATH:$file
 done
 
-if [ -z "$HADOOP_HOME" ]; then
-  echo "No env HADOOP_HOME."
-  exit 1
-fi
-
 if [ -z "$HADOOP_CONF_DIR" ]; then
   echo "No env HADOOP_CONF_DIR."
   exit 1
@@ -85,10 +87,14 @@ JVM_ARGS=" -server \
           -Xloggc:./logs/gc-%t.log"
 
 ARGS=""
-if [ -f ./conf/log4j.properties ]; then
-  ARGS="$ARGS -Dlog4j.configuration=file:./conf/log4j.properties"
+
+LOG_CONF_FILE="./conf/log4j.properties"
+LOG_PATH="./logs/shuffle_server.log"
+
+if [ -f ${LOG_CONF_FILE} ]; then
+  ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
 else
-  echo "Exit with error: $conf/log4j.properties file doesn't exist."
+  echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1
 fi
 

diff --git a/conf/log4j.properties b/conf/log4j.properties
@@ -22,7 +22,7 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
 log4j.appender.RollingAppender=org.apache.log4j.RollingFileAppender
-log4j.appender.RollingAppender.File=./logs/rss.log
+log4j.appender.RollingAppender.File=${log.path}
 log4j.appender.RollingAppender.MaxFileSize=50MB
 log4j.appender.RollingAppender.MaxBackupIndex=10
 log4j.appender.RollingAppender.layout=org.apache.log4j.PatternLayout