HBaseCon 2012 demo scripts

.gitignore

@@ -0,0 +1 @@
+*/credentials.sh

README

@@ -0,0 +1,81 @@
+1. Configuration
+
+   Create a file bin/credentials.sh and configure it according to your
+   account details:
+
+     export AWS_ENDPOINT=us-west-1.ec2.amazonaws.com
+     export EC2_URL=http://$AWS_ENDPOINT/
+     export AWS_ACCOUNT_ID=8015-...
+     export AWS_ACCESS_KEY_ID=AKIA...
+     export AWS_SECRET_ACCESS_KEY=UmU3...
+     export EC2_PRIVATE_KEY=/home/apurtell/.ec2/pk-tm-master.pem
+     export EC2_CERT=/home/apurtell/.ec2/cert-tm-master.pem
+     export EC2_ROOT_SSH_KEY=/home/apurtell/.ec2/root-us-west-1-tm-master.pem
+
+   Configure bin/env.sh
+
+     Set REGION to the region you would like to use.
+
+     Set S3_BUCKET to the bucket where you would like to store the AMI image
+     files.
+
+     Set S3_ACCOUNT to the account that owns S3_BUCKET. Be sure to only use
+     digits (elide the '-').
+
+2. Create an AMI (optional)
+
+   ./bin/create-image [options]
+
+   where [options] can be one or more of:
+
+     -a <arch>    architecture, default x86_64
+     -t <type>    instance type, default m1.small
+
+   This will launch an instance that will build an AMI. 
+
+   When the process of image building is complete the remote instance will
+   still be running. The last step of the build procedure will print out
+   the command you should execute to terminate the build instace.
+
+3. Launch master
+
+   ./bin/launch-master [options]
+
+   where [options] can be one or more of:
+
+     -a <arch>    architecture, default x86_64
+     -t <type>    instance type, default m1.xlarge
+     -m           enable monitoring, default no
+     --ami <id>   AMI ID
+
+   This command will print out the public DNS name of the master instance if
+   successfully launched.
+
+4. Launch slaves
+
+   ./bin/launch-slaves [options] <master> <num slaves>
+
+   where [options] can be one or more of:
+
+     -a <arch>    architecture, default x86_64
+     -t <type>    instance type, default m1.xlarge
+     -m           enable monitoring, default no
+     --ami <id>   AMI ID
+
+   where <master> is the public DNS name of the master instance
+
+   where <num slaves> is the number of slaves to launch
+
+   More slaves can be launched at any time by executing bin/launch-slaves
+   again.
+
+5. SSH to the cluster
+
+   Typically you will want to log in to the master:
+
+   ./bin/ssh-cluster <master>
+
+   where <master> is the public DNS name of the master instance
+
+   Note you can use any public DNS name of any instance in the cluster to log
+   in to any of them.

bin/create-image

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+type=m1.small
+arch=x86_64
+usetar=
+
+while [ $# -gt 0 ] ; do
+  case "$1" in
+    -a | --arch)    arch="$2"; shift;;
+    -t | --type)    type="$2"; shift;;
+    -R | --rpm)     usetar="";;
+    -T | --tar)     usetar="true";;
+    --)             shift; break;;
+    -*)             shift;;
+    *)              break;;
+  esac
+  shift
+done
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+export bin
+if [ -f "$bin"/credentials.sh ] ; then
+  source "$bin"/credentials.sh
+fi
+source "$bin"/env.sh
+
+echo "INSTANCE_TYPE is $type"
+echo "ARCH is $arch"
+echo "IMAGE_VERSION is $IMAGE_VERSION"
+echo "AWS_ENDPOINT is $AWS_ENDPOINT"
+
+AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_ACCOUNT | grep $IMAGE_VERSION-$arch | grep available | awk '{print $2}'`
+
+[ ! -z $AMI_IMAGE ] && echo "AMI already registered, use: ec2-deregister $AMI_IMAGE" && exit 1
+
+BASE_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $BASE_AMI_VERSION | awk '{print $2}'`
+echo "BASE_AMI_IMAGE is $BASE_AMI_IMAGE"
+
+OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $type`
+INSTANCE=`echo $OUTPUT | awk '{print $6}'`
+echo "INSTANCE is $INSTANCE"
+
+echo -n "Polling instance status "
+while true; do
+  printf "."
+  HOSTNAME=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $4}'`
+  if [ ! -z $HOSTNAME ] ; then
+    break;
+  fi
+  sleep 1
+done
+echo
+echo "HOSTNAME is $HOSTNAME"
+echo -n "Polling instance availability "
+while true ; do
+  printf "."
+  REPLY=`ssh $SSH_OPTS "ec2-user@$HOSTNAME" 'echo hello'`
+  if [ ! -z "$REPLY" ] ; then
+    break
+  fi
+  sleep 5
+done
+echo
+
+# Copy setup scripts
+
+echo -n "Copying setup scripts... "
+scp $SSH_OPTS "$bin"/env.sh "ec2-user@$HOSTNAME:/tmp/"
+if [ -f "$bin"/credentials.sh ] ; then
+  scp $SSH_OPTS "$bin"/credentials.sh "ec2-user@$HOSTNAME:/tmp/"
+fi
+scp $SSH_OPTS "$bin"/image/tarball/create-image-remote "ec2-user@$HOSTNAME:/tmp/"
+echo
+
+# Copy private key and certificate (for bundling image)
+
+echo -n "Copying credentials... "
+scp $SSH_OPTS $EC2_PRIVATE_KEY "ec2-user@$HOSTNAME:/tmp/key.pem"
+scp $SSH_OPTS $EC2_CERT "ec2-user@$HOSTNAME:/tmp/cert.pem"
+echo
+
+# Connect to it and run the remote procedure
+echo "Connecting to instance"
+ssh $SSH_OPTS -t -t "ec2-user@$HOSTNAME" "/usr/bin/sudo -i /usr/bin/env ARCH=$arch IMAGE_VERSION=$IMAGE_VERSION /bin/bash /tmp/create-image-remote"
+
+# Register image
+
+echo "Registering new image"
+ec2-register $TOOL_OPTS -n $IMAGE_VERSION-$arch $S3_BUCKET/$IMAGE_VERSION-$arch.manifest.xml
+
+echo "Terminate with: ec2-terminate-instances $INSTANCE"

bin/env.sh

@@ -0,0 +1,48 @@
+# The Amazon EC2 bucket for images
+
+#REGION=${REGION:-us-east-1}
+REGION=${REGION:-us-west-1}
+#REGION=${REGION:-us-west-2}
+#REGION=${REGION:-eu-west-1}
+#REGION=${REGION:-ap-northeast-1}
+#REGION=${REGION:-ap-southeast-1}
+S3_BUCKET=${S3_BUCKET:-tm-bundles-$REGION}
+# Account for bucket
+# We need this because S3 is returning account identifiers instead of bucket
+# names.
+S3_ACCOUNT=801535628028
+
+# The version of HBase to use and the distribution tarball location
+
+export HBASE_VERSION=0.92.1
+export HBASE_URL=http://www.apache.org/dist/hbase/hbase-$HBASE_VERSION/hbase-$HBASE_VERSION-security.tar.gz
+
+# The version of Hadoop to use and the distribution tarball location
+
+export HADOOP_VERSION=1.0.2
+export HADOOP_URL=http://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION-bin.tar.gz
+
+############################################################################
+# Generally, users do not need to edit below
+
+export IMAGE_VERSION="tm-$HBASE_VERSION-$HADOOP_VERSION"
+
+export EC2_URL=${EC2_URL:-https://$REGION.ec2.amazonaws.com}
+
+# SSH options used when connecting to EC2 instances.
+SSH_OPTS=`echo -q -i "$EC2_ROOT_SSH_KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
+
+# EC2 command request timeout (seconds)
+REQUEST_TIMEOUT=300    # 5 minutes
+
+# Global tool options
+TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT" --request-timeout $REQUEST_TIMEOUT`
+
+ARCH=x86_64
+
+BASE_AMI_VERSION="amzn-ami-pv-2012.03.1.${ARCH}-s3"
+
+EPEL_RPM=http://dl.fedoraproject.org/pub/epel/6/${ARCH}/epel-release-6-5.noarch.rpm
+
+JAVA_VERSION=1.7.0_4
+JAVA_RPM=http://tm-files-west.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${ARCH}.rpm