Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fix issue #1

Update version to 0.20.6.

Save a few bytes in hbase-ec2-init-remote.sh (EC2 limits this to <= 16KB)
  • Loading branch information...
commit e2384222afdad33d49cdd60ede48604d76ac1600 1 parent 7351901
Andrew Purtell authored
2  bin/hbase-ec2-env.sh
@@ -34,7 +34,7 @@ EC2_CERT=
34 34
 EC2_ROOT_SSH_KEY=
35 35
 
36 36
 # The version of HBase to use.
37  
-HBASE_VERSION=0.20.4
  37
+HBASE_VERSION=0.20.6
38 38
 
39 39
 # The version of Hadoop to use.
40 40
 HADOOP_VERSION=0.20.2
189  bin/hbase-ec2-init-remote.sh
@@ -19,10 +19,8 @@
19 19
 # See the License for the specific language governing permissions and
20 20
 # limitations under the License.
21 21
 #
22  
-
23 22
 # Script that is run on each EC2 instance on boot. It is passed in the EC2 user
24 23
 # data, so should not exceed 16K in size.
25  
-
26 24
 MASTER_HOST="%MASTER_HOST%"
27 25
 ZOOKEEPER_QUORUM="%ZOOKEEPER_QUORUM%"
28 26
 EXTRA_PACKAGES="%EXTRA_PACKAGES%"
@@ -35,92 +33,78 @@ HADOOP_HOME=`ls -d /usr/local/hadoop-*`
35 33
 HADOOP_VERSION=`echo $HADOOP_HOME | cut -d '-' -f 2`
36 34
 HBASE_HOME=`ls -d /usr/local/hbase-*`
37 35
 HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2`
38  
-
39 36
 export USER="root"
40  
-
41 37
 # up file-max
42 38
 sysctl -w fs.file-max=65535
43  
-
44 39
 # up ulimits
45 40
 echo "root soft nofile 65535" >> /etc/security/limits.conf
46 41
 echo "root hard nofile 65535" >> /etc/security/limits.conf
47 42
 ulimit -n 65535
48  
-
49 43
 # up epoll limits; ok if this fails, only valid for kernels 2.6.27+
50 44
 sysctl -w fs.epoll.max_user_instances=65535 > /dev/null 2>&1
51  
-
52 45
 [ ! -f /etc/hosts ] &&  echo "127.0.0.1 localhost" > /etc/hosts
53  
-
54 46
 # Extra packages
55  
-
56 47
 if [ "$EXTRA_PACKAGES" != "" ] ; then
57  
-  # format should be <repo-descriptor-URL> <package1> ... <packageN>
58  
-  pkg=( $EXTRA_PACKAGES )
59  
-  wget -nv -O /etc/yum.repos.d/user.repo ${pkg[0]}
60  
-  yum -y update yum
61  
-  yum -y install ${pkg[@]:1}
  48
+ # format should be <repo-descriptor-URL> <package1> ... <packageN>
  49
+ pkg=( $EXTRA_PACKAGES )
  50
+ wget -nv -O /etc/yum.repos.d/user.repo ${pkg[0]}
  51
+ yum -y update yum
  52
+ yum -y install ${pkg[@]:1}
62 53
 fi
63  
-
64  
-# Ganglia
65  
-
66  
-if [ "$IS_MASTER" = "true" ]; then
67  
-  sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
68  
-         -e "s|\( *bind *=.*\)|#\1|" \
69  
-         -e "s|\( *mute *=.*\)|  mute = yes|" \
70  
-         -e "s|\( *location *=.*\)|  location = \"master-node\"|" \
71  
-         /etc/gmond.conf
72  
-  mkdir -p /mnt/ganglia/rrds
73  
-  chown -R ganglia:ganglia /mnt/ganglia/rrds
74  
-  rm -rf /var/lib/ganglia; cd /var/lib; ln -s /mnt/ganglia ganglia; cd
75  
-  service gmond start
76  
-  service gmetad start
77  
-  apachectl start
78  
-else
79  
-  sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
80  
-         -e "s|\( *bind *=.*\)|#\1|" \
81  
-         -e "s|\(udp_send_channel {\)|\1\n  host=$MASTER_HOST|" \
82  
-         /etc/gmond.conf
83  
-  service gmond start
84  
-fi
85  
-
86 54
 # Reformat sdb as xfs
87 55
 umount /mnt
88 56
 mkfs.xfs -f /dev/sdb
89 57
 mount -o noatime /dev/sdb /mnt
90  
-
91 58
 # Probe for additional instance volumes
92  
-
93 59
 # /dev/sdb as /mnt is always set up by base image
94 60
 DFS_NAME_DIR="/mnt/hadoop/dfs/name"
95 61
 DFS_DATA_DIR="/mnt/hadoop/dfs/data"
96 62
 i=2
97 63
 for d in c d e f g h i j k l m n o p q r s t u v w x y z; do
98  
-  m="/mnt${i}"
99  
-  mkdir -p $m
100  
-  mkfs.xfs -f /dev/sd${d}
101  
-  if [ $? -eq 0 ] ; then
102  
-    mount -o noatime /dev/sd${d} $m > /dev/null 2>&1
103  
-    if [ $i -lt 3 ] ; then # no more than two namedirs
104  
-      DFS_NAME_DIR="${DFS_NAME_DIR},${m}/hadoop/dfs/name"
105  
-    fi
106  
-    DFS_DATA_DIR="${DFS_DATA_DIR},${m}/hadoop/dfs/data"
107  
-    i=$(( i + 1 ))
  64
+ m="/mnt${i}"
  65
+ mkdir -p $m
  66
+ mkfs.xfs -f /dev/sd${d}
  67
+ if [ $? -eq 0 ] ; then
  68
+  mount -o noatime /dev/sd${d} $m > /dev/null 2>&1
  69
+  if [ $i -lt 3 ] ; then # no more than two namedirs
  70
+   DFS_NAME_DIR="${DFS_NAME_DIR},${m}/hadoop/dfs/name"
108 71
   fi
  72
+  DFS_DATA_DIR="${DFS_DATA_DIR},${m}/hadoop/dfs/data"
  73
+  i=$(( i + 1 ))
  74
+ fi
109 75
 done
110  
-
  76
+# Set up Ganglia
  77
+if [ "$IS_MASTER" = "true" ]; then
  78
+ sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
  79
+  -e "s|\( *bind *=.*\)|#\1|" \
  80
+  -e "s|\( *mute *=.*\)|  mute = yes|" \
  81
+  -e "s|\( *location *=.*\)|  location = \"master-node\"|" \
  82
+  /etc/gmond.conf
  83
+ mkdir -p /mnt/ganglia/rrds
  84
+ chown -R ganglia:ganglia /mnt/ganglia/rrds
  85
+ rm -rf /var/lib/ganglia; cd /var/lib; ln -s /mnt/ganglia ganglia; cd
  86
+ service gmond start
  87
+ service gmetad start
  88
+ apachectl start
  89
+else
  90
+ sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
  91
+  -e "s|\( *bind *=.*\)|#\1|" \
  92
+  -e "s|\(udp_send_channel {\)|\1\n  host=$MASTER_HOST|" \
  93
+  /etc/gmond.conf
  94
+ service gmond start
  95
+fi
111 96
 # Hadoop configuration
112  
-
113 97
 cat > $HADOOP_HOME/conf/core-site.xml <<EOF
114 98
 <?xml version="1.0"?>
115 99
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
116 100
 <configuration>
117 101
 <property>
118  
-  <name>hadoop.tmp.dir</name>
119  
-  <value>/mnt/hadoop</value>
  102
+ <name>hadoop.tmp.dir</name>
  103
+ <value>/mnt/hadoop</value>
120 104
 </property>
121 105
 <property>
122  
-  <name>fs.default.name</name>
123  
-  <value>hdfs://$MASTER_HOST:8020</value>
  106
+ <name>fs.default.name</name>
  107
+ <value>hdfs://$MASTER_HOST:8020</value>
124 108
 </property>
125 109
 </configuration>
126 110
 EOF
@@ -129,24 +113,24 @@ cat > $HADOOP_HOME/conf/hdfs-site.xml <<EOF
129 113
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
130 114
 <configuration>
131 115
 <property>
132  
-  <name>fs.default.name</name>
133  
-  <value>hdfs://$MASTER_HOST:8020</value>
  116
+ <name>fs.default.name</name>
  117
+ <value>hdfs://$MASTER_HOST:8020</value>
134 118
 </property>
135 119
 <property>
136  
-  <name>dfs.name.dir</name>
137  
-  <value>$DFS_NAME_DIR</value>
  120
+ <name>dfs.name.dir</name>
  121
+ <value>$DFS_NAME_DIR</value>
138 122
 </property>
139 123
 <property>
140  
-  <name>dfs.data.dir</name>
141  
-  <value>$DFS_DATA_DIR</value>
  124
+ <name>dfs.data.dir</name>
  125
+ <value>$DFS_DATA_DIR</value>
142 126
 </property>
143 127
 <property>
144  
-  <name>dfs.datanode.handler.count</name>
145  
-  <value>10</value>
  128
+ <name>dfs.datanode.handler.count</name>
  129
+ <value>10</value>
146 130
 </property>
147 131
 <property>
148  
-  <name>dfs.datanode.max.xcievers</name>
149  
-  <value>10000</value>
  132
+ <name>dfs.datanode.max.xcievers</name>
  133
+ <value>10000</value>
150 134
 </property>
151 135
 </configuration>
152 136
 EOF
@@ -155,18 +139,18 @@ cat > $HADOOP_HOME/conf/mapred-site.xml <<EOF
155 139
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
156 140
 <configuration>
157 141
 <property>
158  
-  <name>mapred.job.tracker</name>
159  
-  <value>$MASTER_HOST:8021</value>
  142
+ <name>mapred.job.tracker</name>
  143
+ <value>$MASTER_HOST:8021</value>
160 144
 </property>
161 145
 <property>
162  
-  <name>io.compression.codecs</name>
163  
-  <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
  146
+ <name>io.compression.codecs</name>
  147
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
164 148
 </property>
165 149
 </configuration>
166 150
 EOF
167 151
 # Update classpath to include HBase jars and config
168 152
 cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
169  
-HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/zookeeper-3.3.0.jar:$HBASE_HOME/conf"
  153
+HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/zookeeper-3.3.1.jar:$HBASE_HOME/conf"
170 154
 EOF
171 155
 # Configure Hadoop for Ganglia
172 156
 cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
@@ -180,59 +164,45 @@ mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
180 164
 mapred.period=10
181 165
 mapred.servers=$MASTER_HOST:8649
182 166
 EOF
183  
-
184 167
 # HBase configuration
185  
-
186 168
 cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
187 169
 <?xml version="1.0"?>
188 170
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
189 171
 <configuration>
190 172
 <property>
191  
-  <name>hbase.rootdir</name>
192  
-  <value>hdfs://$MASTER_HOST:8020/hbase</value>
193  
-</property>
194  
-<property>
195  
-  <name>hbase.cluster.distributed</name>
196  
-  <value>true</value>
197  
-</property>
198  
-<property>
199  
-  <name>hbase.zookeeper.quorum</name>
200  
-  <value>$ZOOKEEPER_QUORUM</value>
201  
-</property>
202  
-<property>
203  
-  <name>hbase.regionserver.handler.count</name>
204  
-  <value>100</value>
  173
+ <name>hbase.rootdir</name>
  174
+ <value>hdfs://$MASTER_HOST:8020/hbase</value>
205 175
 </property>
206 176
 <property>
207  
-  <name>hbase.hregion.memstore.block.multiplier</name>
208  
-  <value>3</value>
  177
+ <name>hbase.cluster.distributed</name>
  178
+ <value>true</value>
209 179
 </property>
210 180
 <property>
211  
-  <name>hbase.hstore.blockingStoreFiles</name>
212  
-  <value>15</value>
  181
+ <name>hbase.zookeeper.quorum</name>
  182
+ <value>$ZOOKEEPER_QUORUM</value>
213 183
 </property>
214 184
 <property>
215  
-  <name>dfs.replication</name>
216  
-  <value>3</value>
  185
+ <name>hbase.regionserver.handler.count</name>
  186
+ <value>100</value>
217 187
 </property>
218 188
 <property>
219  
-  <name>dfs.client.block.write.retries</name>
220  
-  <value>20</value>
  189
+ <name>dfs.replication</name>
  190
+ <value>3</value>
221 191
 </property>
222 192
 <property>
223  
-  <name>zookeeper.session.timeout</name>
224  
-  <value>60000</value>
  193
+ <name>zookeeper.session.timeout</name>
  194
+ <value>60000</value>
225 195
 </property>
226 196
 <property>
227  
-  <name>hbase.tmp.dir</name>
228  
-  <value>/mnt/hbase</value>
  197
+ <name>hbase.tmp.dir</name>
  198
+ <value>/mnt/hbase</value>
229 199
 </property>
230 200
 </configuration>
231 201
 EOF
232 202
 # Override JVM options
233 203
 cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
234  
-export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseConcMarkSweepGC -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
235  
-export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
  204
+export HBASE_MASTER_OPTS="-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
  205
+export HBASE_REGIONSERVER_OPTS="-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
236 206
 EOF
237 207
 # Configure HBase for Ganglia
238 208
 cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
@@ -246,19 +216,16 @@ jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
246 216
 jvm.period=10
247 217
 jvm.servers=$MASTER_HOST:8649
248 218
 EOF
249  
-
250 219
 mkdir -p /mnt/hadoop/logs /mnt/hbase/logs
251  
-
252 220
 if [ "$IS_MASTER" = "true" ]; then
253  
-  # only format on first boot
254  
-  [ ! -e /mnt/hadoop/dfs/name ] && "$HADOOP_HOME"/bin/hadoop namenode -format
255  
-  "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode
256  
-  "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
257  
-  "$HBASE_HOME"/bin/hbase-daemon.sh start master
  221
+ # only format on first boot
  222
+ [ ! -e /mnt/hadoop/dfs/name ] && "$HADOOP_HOME"/bin/hadoop namenode -format
  223
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode
  224
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
  225
+ "$HBASE_HOME"/bin/hbase-daemon.sh start master
258 226
 else
259  
-  "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
260  
-  "$HBASE_HOME"/bin/hbase-daemon.sh start regionserver
261  
-  "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
  227
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
  228
+ "$HBASE_HOME"/bin/hbase-daemon.sh start regionserver
  229
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
262 230
 fi
263  
-
264 231
 rm -f /var/ec2/ec2-run-user-data.*

0 notes on commit e238422

Please sign in to comment.
Something went wrong with that request. Please try again.