Creating 2 jobs that contain various hadoop, hbase and opentsdb compo…

…nents.

Change-Id: I08f0ad1ee21d87e573b230401e0b0d0188b9e22b

config/blobs.yml

@@ -87,10 +87,6 @@ hbase/native-lib.tar.gz:
   object_id: eyJvaWQiOiI0ZTRlNzhiY2E2MWUxMjEwMDRlNGU3ZDUxZDk1MGUwNGYzMDY4%0AZTQ5ODhjOCIsInNpZyI6IkFlK1c2bGlOVFVqSGk2alB0Q3IzUlBOR3pvdz0i%0AfQ==%0A
   sha: 8d35c05ce07f0af4a19926fb1497ba101b4116ba
   size: 546495
-hbase/hbase-0.90.3-cdh3u1.tar.gz:
-  object_id: eyJvaWQiOiI0ZTRlNzhiY2E2MWUxMjIwMDRlNGU4ZWM2YjQ0YjYwNGYzMDY4%0AZWEyNmVlNyIsInNpZyI6IlMvZ1R5ZXlRdHhLZUxqSGU0QUU5WmErd1F3WT0i%0AfQ==%0A
-  sha: 34e7de45ed6c36f618e9625128422023868c5e51
-  size: 35113844
 postgresql/postgresql-9.0-x86_64.tar.gz:
   object_id: eyJvaWQiOiI0ZTRlNzhiY2ExMWUxMjEwMDRlNGU3ZDUxMWY4MjEwNGYzMDY4%0AZmI3MTJlNCIsInNpZyI6IkMzS0hrcTAxWnZSVU9wQVI2OWFXWXFlR1FVST0i%0AfQ==%0A
   sha: 1c777413ce355ebf79c5786e7dbc5a060883986f
@@ -579,3 +575,11 @@ warden/libnl1_1.1-5build1_amd64.deb:
   object_id: eyJvaWQiOiI0ZTRlNzhiY2EyMWUxMjEyMDRlNGU4NmVlMTUxYmMwNTAwODk0%0ANDljZTAwNyIsInNpZyI6IndsQ3ZVVVBjVjVrZUJXOWkwSlA5UHQ4ZVNHYz0i%0AfQ==%0A
   sha: 2c6c2dfa973ba4fa24dcc45153c07eb622c360fa
   size: 143038
+hadoop/hadoop-0.20.2-cdh3u4.tar.gz:
+  object_id: eyJvaWQiOiI0ZTRlNzhiY2E1MWUxMjIyMDRlNGU5ODYzZjI4ZjMwNGZiZDAx%0AODQwN2YwYiIsInNpZyI6ImtrWDA3SXVFVzNiTmNPSHhqMGNhd0Nmb2I0RT0i%0AfQ==%0A
+  sha: 1a9e6437b4b7f0bdbfdd3bd9170a33238e151f12
+  size: 70157279
+hbase/hbase-0.90.6-cdh3u4-vmware-patched.tar.gz:
+  object_id: eyJvaWQiOiI0ZTRlNzhiY2ExMWUxMjIyMDRlNGU5ODYzOGI3NjMwNTAxMDVh%0AYzVjNTkwMSIsInNpZyI6IldlMi9yRjRybFZ3Y1ptR3FwWklzcHFZT2RiUT0i%0AfQ==%0A
+  sha: 4290e2a35fc54d34d8ea4e0559e6de6e485322d4
+  size: 36978703

jobs/hbase_master/monit

@@ -0,0 +1,19 @@
+check process hadoop_namenode
+  with pidfile /var/vcap/sys/run/hbase_master/hadoop-vcap-namenode.pid
+  start program "/var/vcap/jobs/hbase_master/bin/hadoop_namenode_ctl start"
+  stop program "/var/vcap/jobs/hbase_master/bin/hadoop_namenode_ctl stop"
+  group vcap
+
+check process hbase_zookeeper
+  with pidfile /var/vcap/sys/run/hbase_master/hbase-vcap-zookeeper.pid
+  start program "/var/vcap/jobs/hbase_master/bin/hbase_zookeeper_ctl start"
+  stop program "/var/vcap/jobs/hbase_master/bin/hbase_zookeeper_ctl stop"
+  depends on hadoop_namenode
+  group vcap
+
+check process hbase_master
+  with pidfile /var/vcap/sys/run/hbase_master/hbase-vcap-master.pid
+  start program "/var/vcap/jobs/hbase_master/bin/hbase_master_ctl start"
+  stop program "/var/vcap/jobs/hbase_master/bin/hbase_master_ctl stop"
+  depends on hbase_zookeeper
+  group vcap

jobs/hbase_master/spec

@@ -0,0 +1,19 @@
+---
+name: hbase_master
+templates:
+  hadoop.core-site.xml.erb: config/hadoop/core-site.xml
+  hadoop.hdfs-site.xml: config/hadoop/hdfs-site.xml
+  hadoop.slaves.erb: config/hadoop/slaves
+  hadoop_namenode_ctl: bin/hadoop_namenode_ctl
+  hbase.hadoop-metrics.properties: config/hbase/hadoop-metrics.properties
+  hbase.hbase-env.sh: config/hbase/hbase-env.sh
+  hbase.hbase-site.xml.erb: config/hbase/hbase-site.xml
+  hbase.log4j.properties: config/hbase/log4j.properties
+  hbase.regionservers.erb: config/hbase/regionservers
+  hbase_master_ctl: bin/hbase_master_ctl
+  hbase_zookeeper_ctl: bin/hbase_zookeeper_ctl
+packages:
+- common
+- dea_jvm
+- hadoop
+- hbase

jobs/hbase_master/templates/hadoop.core-site.xml.erb

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://<%= properties.hbase_master.address %>:<%= properties.hbase_master.hadoop_namenode.port %></value>
+    <description>URI of NameNode.</description>
+  </property>
+</configuration>

jobs/hbase_master/templates/hadoop.hdfs-site.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>dfs.name.dir</name>
+    <value>/var/vcap/store/hbase_master/name</value>
+    <description>Path on the local filesystem where the NameNode stores the namespace and transactions logs persistently.</description>
+  </property>
+  <property>
+    <name>dfs.data.dir</name>
+    <value>/var/vcap/store/hbase_master/data</value>
+    <description>Comma separated list of paths on the local filesystem of where a DataNode should store its blocks.</description>
+  </property>
+</configuration>

jobs/hbase_master/templates/hadoop.slaves.erb

@@ -0,0 +1,2 @@
+<% for address in properties.hbase_slave.addresses %><%= address %>
+<% end %>

jobs/hbase_master/templates/hadoop_namenode_ctl

@@ -0,0 +1,70 @@
+#!/bin/bash -e
+
+HADOOP_PACKAGE_DIR="/var/vcap/packages/hadoop"
+JOBS_DIR="/var/vcap/jobs/hbase_master"
+LOG_DIR="/var/vcap/sys/log/hbase_master"
+RUN_DIR="/var/vcap/sys/run/hbase_master"
+
+HADOOP_BIN="${HADOOP_PACKAGE_DIR}/bin"
+HADOOP_DATA_DIR="/var/vcap/store/hbase_master"
+MASTER_HOSTNAME="<%= properties.hbase_master.hostname %>"
+PIDFILE="${RUN_DIR}/hadoop-vcap-namenode.pid"
+TMP_DIR="/var/vcap/store/hbase"
+
+export HADOOP_CONF_DIR="${JOBS_DIR}/config/hadoop"
+export HADOOP_IDENT_STRING="vcap"
+export HADOOP_LOG_DIR=${LOG_DIR}
+export HADOOP_NAMENODE_USER="vcap"
+export HADOOP_PID_DIR=${RUN_DIR}
+export JAVA_HOME="/var/vcap/packages/dea_jvm"
+
+source /var/vcap/packages/common/utils.sh
+
+case $1 in
+
+  start)
+    # hadoop-daemon.sh has its own pid guard,
+    # we use ours for consistency anyway
+    pid_guard $PIDFILE "hadoop_namenode"
+
+    mkdir -p $RUN_DIR
+    mkdir -p $LOG_DIR
+    mkdir -p $HADOOP_DATA_DIR
+    mkdir -p $TMP_DIR
+
+    chown vcap:vcap $RUN_DIR
+    chown vcap:vcap $LOG_DIR
+    chown vcap:vcap $HADOOP_DATA_DIR
+    chown vcap:vcap $TMP_DIR
+    # Set maximum number of open file descriptors.
+    ulimit -n 32768
+    # Set maximum number of processes available to a single user.
+    ulimit -u 32000
+
+    if [ ! -d /var/vcap/store/hbase_master/name ]
+    then
+      cd "${HADOOP_PACKAGE_DIR}/bin"
+      ./hadoop --config $HADOOP_CONF_DIR namenode -format
+    fi
+
+    # hadoop ctl scripts manage their pidfiles,
+    # so we don't attempt to write ours
+    exec chpst -u vcap:vcap $HADOOP_BIN/hadoop-daemon.sh --config $HADOOP_CONF_DIR start namenode \
+         >>$LOG_DIR/hadoop_namenode_start.stdout.log \
+         2>>$LOG_DIR/hadoop_namenode_start.stderr.log
+
+    ;;
+
+  stop)
+    exec chpst -u vcap:vcap $HADOOP_BIN/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop namenode \
+         >>$LOG_DIR/hadoop_namenode_stop.stdout.log \
+         2>>$LOG_DIR/hadoop_namenode_stop.stderr.log
+
+    ;;
+
+  *)
+    echo "Usage: hadoop_namenode_ctl {start|stop}"
+
+    ;;
+
+esac

jobs/hbase_master/templates/hbase.hadoop-metrics.properties

@@ -0,0 +1,58 @@
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+
+# Configuration of the "hbase" context for null
+hbase.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.period=10
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.period=10
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.period=10
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649

jobs/hbase_master/templates/hbase.hbase-env.sh

@@ -0,0 +1,76 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# The java implementation to use.  Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements.  Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=3072
+
+# Extra Java runtime options.
+# Below are what we set by default.  May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode"
+
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101 -javaagent:lib/HelloWorldAgent.jar"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run.  $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# Extra ssh options.  Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored.  $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes.  See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands.  Unset by default.  This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true

jobs/hbase_master/templates/hbase.hbase-site.xml.erb

@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>hdfs://<%= properties.hbase_master.address %>:<%= properties.hbase_master.hadoop_namenode.port %>/hbase</value>
+  </property>
+  <property>
+    <name>hbase.master.port</name>
+    <value><%= properties.hbase_master.hbase_master.port %></value>
+  </property>
+  <property>
+    <name>hbase.master.info.port</name>
+    <value><%= properties.hbase_master.hbase_master.webui_port %></value>
+  </property>
+  <property>
+    <name>hbase.use_ips</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.master.dns.interface</name>
+    <value>eth0</value>
+  </property>
+  <property>
+    <name>hbase.regionserver.dns.interface</name>
+    <value>eth0</value>
+  </property>
+  <property>
+    <name>hbase.regionserver.port</name>
+    <value><%= properties.hbase_slave.hbase_regionserver.port %></value>
+  </property>
+  <property>
+    <name>hbase.tmp.dir</name>
+    <value>/var/vcap/store/hbase</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value><%= properties.hbase_master.address %></value>
+  </property>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+    <description>The mode the cluster will be in. Possible values are
+      false: standalone and pseudo-distributed setups with managed Zookeeper
+      true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
+    </description>
+  </property>
+</configuration>

jobs/hbase_master/templates/hbase.log4j.properties

@@ -0,0 +1,13 @@
+log4j.rootLogger=INFO,FA
+
+log4j.threshhold=ALL
+
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.File=/var/vcap/sys/log/hbase_master/hbase_master.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO

jobs/hbase_master/templates/hbase.regionservers.erb

@@ -0,0 +1,2 @@
+<% for address in properties.hbase_slave.addresses %><%= address %>
+<% end %>