Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
46 lines (37 sloc) 1.37 KB
#!/bin/bash
################################################################################
#
# Job script to spin up a semi-persistent Hadoop cluster on FutureGrid Sierra
#
# Glenn K. Lockwood, San Diego Supercomputer Center June 2013
#
################################################################################
#PBS -N hadoopcluster
#PBS -l nodes=4:ppn=8
#PBS -l walltime=12:00:00
#PBS -q batch
#PBS -j oe
#PBS -o hadoopcluster.log
#PBS -V
module add java
cd $PBS_O_WORKDIR
export MY_HADOOP_HOME="/opt/myHadoop"
source $MY_HADOOP_HOME/bin/setenv.sh
export HADOOP_CONF_DIR=$PBS_O_WORKDIR/$PBS_JOBID
if [ $USE_INFINIBAND ]; then
### Enable Hadoop over Infiniband (add "ib" suffix to node hostnames).
### 6/24/2013: TCP over IB is not currently configured properly on Sierra
OLD_PBS_NODEFILE=$PBS_NODEFILE
export PBS_NODEFILE=$(mktemp)
sed -e 's/$/ib/g' $OLD_PBS_NODEFILE > $PBS_NODEFILE
$MY_HADOOP_HOME/bin/pbs-configure.sh -n 4 -c $HADOOP_CONF_DIR || exit 1
export PBS_NODEFILE=$OLD_PBS_NODEFILE
else
$MY_HADOOP_HOME/bin/pbs-configure.sh -n 4 -c $HADOOP_CONF_DIR || exit 1
fi
$HADOOP_HOME/bin/hadoop --config $HADOOP_CONF_DIR namenode -format
$HADOOP_HOME/bin/start-all.sh
sleep $((12*3600-180))
$HADOOP_HOME/bin/stop-all.sh
cp -Lr $HADOOP_LOG_DIR $PBS_O_WORKDIR/hadoop-logs.$PBS_JOBID
$MY_HADOOP_HOME/bin/pbs-cleanup.sh -n 4 -c $HADOOP_CONF_DIR