alestic-git-build-ami

#!/bin/bash -ex
#
# alestic-git-build-ami
#
# Build new AMI for an Alestic Git Server.
#
# This expects to run on an EC2 instance.
#

# Defaults

# Brand used in AMI name and description
brand="My"

# Size of AMI file system
size=8 # GB

# Ubuntu release
codename=precise

# AMI name timestamp
now=$(date -u +%Y%m%d-%H%M)

# aws-cli credentials
export AWS_CONFIG_FILE=/mnt/aws.config

# Command line options
while [ $# -gt 0 ]; do
  case $1 in
    --brand)       brand=$2;     shift 2 ;;
    --size)        size=$2;      shift 2 ;;
    --codename)    codename=$2;  shift 2 ;;
    --now)         now=$2;       shift 2 ;;
    *)             echo "$0: Unrecognized option: $1" >&2; exit 1;
  esac
done

# Setup
case $codename in
  maverick)   release=10.10     ;;
  natty)      release=11.04     ;;
  oneiric)    release=11.10     ;;
  precise)    release=12.04     ;;
  quantal)    release=12.10     ;;
  trusty)     release=14.04     ;;
  *)          echo "$0: Unrecognized codename: $codename" >&2; exit 1;
esac

if [ $(uname -m) = 'x86_64' ]; then
  arch=x86_64
  arch2=amd64
  ephemeraldev=/dev/sdb
else
  arch=i386
  arch2=i386
  ephemeraldev=/dev/sda2
fi

name="${brand,}-git-server-ubuntu-$release-$codename-$arch2-$now"
description="$brand Git Server - Ubuntu $release ${codename^} $arch2 $now"

#export EC2_CERT=$(echo /mnt/cert-*.pem)
#export EC2_PRIVATE_KEY=$(echo /mnt/pk-*.pem)

imagename=$codename-server-cloudimg-$arch2
imageurl=http://cloud-images.ubuntu.com/$codename/current/$imagename.tar.gz
amisurl=http://cloud-images.ubuntu.com/query/$codename/server/released.current.txt
zoneurl=http://instance-data/latest/meta-data/placement/availability-zone

# Sometimes, instance-data does not resolve right away.
# Loop to give it some time.
while zone=$(wget -qO- $zoneurl);
      test "$zone" = ""
do
  sleep 30
done; echo " $zone"

region=$(echo $zone | perl -pe 's/.$//')
akiid=$(wget -qO- $amisurl | egrep "ebs.$arch2.$region.*paravirtual" | cut -f9)
#ariid=$(wget -qO- $amisurl | egrep "ebs.$arch2.$region.*paravirtual" | cut -f10)

# Set the default region for aws cli to our current region
export AWS_DEFAULT_REGION=$region

# Update and install Ubuntu packages
export DEBIAN_FRONTEND=noninteractive
sudo perl -pi -e 's/^# *(deb .*multiverse)$/$1/' /etc/apt/sources.list
sudo apt-get update
sudo -E apt-get upgrade -y
sudo apt-get install -y python-pip
sudo pip install awscli

# Download base Ubuntu server image built by Canonical
image=/mnt/$imagename.img
imagedir=/mnt/$codename-cloudimg-$arch2
wget -qO- $imageurl |
  sudo tar xzf - -C /mnt
sudo mkdir -p $imagedir
sudo mount -o loop $image $imagedir

# Allow network access from chroot environment
sudo mkdir -p $imagedir/run/resolvconf
sudo cp /etc/resolv.conf $imagedir/run/resolvconf/resolv.conf
#sudo cp /etc/resolv.conf $imagedir/etc/resolvconf/resolv.conf.d/tail

# We used XFS for the target root file system
#sudo perl -pi -e 's%(\t/\t)ext4(\t)%${1}xfs${2}%' $imagedir/etc/fstab

# Upgrade and install packages on the target file system
sudo chroot $imagedir mount -t proc none /proc
#sudo chroot $imagedir mount -t devpts none /dev/pts
cat <<EOF | sudo tee $imagedir/usr/sbin/policy-rc.d > /dev/null
#!/bin/sh
exit 101
EOF
sudo chmod 755 $imagedir/usr/sbin/policy-rc.d
DEBIAN_FRONTEND=noninteractive
sudo perl -pi -e 's/^# *(deb .*multiverse)$/$1/' \
  $imagedir/etc/apt/sources.list                 \
  $imagedir/etc/cloud/templates/sources.list.ubuntu.tmpl
#sudo chroot $imagedir add-apt-repository ppa:alestic
sudo chroot $imagedir apt-get update
sudo -E chroot $imagedir apt-get dist-upgrade -y

#### BEGIN alestic-git

gituser=git

# Install software
sudo -E chroot $imagedir    \
  apt-get install -y        \
    coreutils               \
    git-core                \
    nginx-light             \
    unattended-upgrades     \

# Apply Ubuntu security patches daily
cat <<EOF | sudo tee $imagedir/etc/apt/apt.conf.d/10periodic > /dev/null
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Download-Upgradeable-Packages "1";
APT::Periodic::AutocleanInterval "7";
APT::Periodic::Unattended-Upgrade "1";
EOF

# Create git user with hardcoded UID
sudo chroot $imagedir       \
  addgroup --gid 500 $gituser
sudo chroot $imagedir       \
  adduser                   \
    --home /home/$gituser   \
    --uid 500               \
    --gid 500               \
    --disabled-password     \
    --gecos gitolite        \
    $gituser

echo 'PATH="$HOME/bin:$PATH"' |
  sudo tee -a $imagedir/home/$gituser/.bashrc > /dev/null

# Install git as the git user in the chroot
sudo chroot $imagedir sudo -i -u $gituser <<EOF
  # The Ubuntu gitolite package is a bit out of date,
  # so install gitolite from source on github
  git clone --branch master git://github.com/sitaramc/gitolite

  # We'll install gitolite in local directories
  mkdir -p                    \
    /home/$gituser/bin                 \
    /home/$gituser/share/gitolite/conf \
    /home/$gituser/share/gitolite/hooks
  PATH=$PATH:/home/$gituser/bin

  # Install gitolite
  gitolite/install -ln /home/$gituser/bin
EOF

# Install init script and config
sudo cp /mnt/alestic-git-init $imagedir/usr/bin/alestic-git-init
sudo cp /mnt/alestic-git.conf $imagedir/etc/init/alestic-git.conf

# git version
git_version=$(sudo chroot $imagedir git --version)

# gitolite version and branch
gitolite_version=$(sudo chroot $imagedir bash -c \
  "cd /home/$gituser/gitolite; git describe --tags")
gitolite_branch=$(sudo chroot $imagedir bash -c \
  "cd /home/$gituser/gitolite; git branch | grep '*' | cut -f2 -d' '")

#### END alestic-git

# Clean up chroot environment
sudo chroot $imagedir umount /proc
#sudo chroot $imagedir umount /dev/pts
sudo rm -f $imagedir/usr/sbin/policy-rc.d
sudo rm -f $imagedir/run/resolvconf/resolv.conf

# Create and mount temporary EBS volume with file system to hold new AMI image
volumeid=$(sudo -E aws ec2 create-volume \
             --availability-zone "$zone" \
             --size "$size"              \
             --volume-type gp2           \
             --output text               \
             --query 'VolumeId' )
while sudo -E aws ec2 describe-volumes \
        --volume-id "$volumeid" \
        --output text \
        --query 'Volumes[*].State' |
      grep -v -q available
  do sleep 3; done
instance_id=$(wget -qO- http://instance-data/latest/meta-data/instance-id)
sudo -E aws ec2 attach-volume   \
   --device /dev/sdi            \
   --instance-id "$instance_id" \
   --volume-id "$volumeid"      \
   --output text                \
   --query 'State'
dev=/dev/xvdi
while [ ! -e $dev ]
  do sleep 3; done
sudo mkfs.ext4 -L cloudimg-rootfs $dev
ebsimagedir=$imagedir-ebs
sudo mkdir $ebsimagedir
sudo mount $dev $ebsimagedir

# Copy file system from temporary rootdir to EBS volume
sudo tar -cSf - -C $imagedir . | sudo tar xvf - -C $ebsimagedir
sudo umount $imagedir
sudo umount $ebsimagedir
sudo -E aws ec2 detach-volume \
  --volume-id "$volumeid" \
  --output text \
  --query 'State'
while sudo -E aws ec2 describe-volumes \
        --volume-id "$volumeid" \
        --output text \
        --query 'Volumes[*].State' |
      grep -v -q available
  do sleep 3; done
snapshotid=$(sudo -E aws ec2 create-snapshot \
               --description "$name" \
               --volume-id "$volumeid" \
               --output text \
               --query 'SnapshotId' )
while sudo -E aws ec2 describe-snapshots \
        --snapshot-id "$snapshotid" \
        --output text \
        --query 'Snapshots[*].State' |
      grep pending
  do sleep 10; done

# Register the snapshot as a new AMI
block_device_mapping=$(cat <<EOF
[
  {
    "DeviceName": "/dev/sda1",
    "Ebs": {
      "DeleteOnTermination": true, 
      "SnapshotId": "$snapshotid",
      "VolumeSize": $size,
      "VolumeType": "gp2"
    }
  }, {
    "DeviceName": "$ephemeraldev",
    "VirtualName": "ephemeral0"
  }
]
EOF
)
amiid=$(sudo -E aws ec2 register-image            \
  --name "$name"                                  \
  --description "$description"                    \
  --architecture "$arch"                          \
  --kernel-id "$akiid"                            \
  --block-device-mapping "$block_device_mapping"  \
  --root-device-name "/dev/sda1"                  \
  --output text                                   \
  --query 'ImageId'
)

sudo -E aws ec2 delete-volume \
  --volume-id "$volumeid" \
  --output text

cat <<EOF
AMI: $amiid $codename $region $arch2

ami id:       $amiid
aki id:       $akiid
region:       $region ($zone)
architecture: $arch ($arch2)
os:           Ubuntu $release $codename
name:         $name
description:  $description
EBS volume:   $volumeid (deleted)
EBS snapshot: $snapshotid
git:          $git_version
gitolite:     $gitolite_version ($gitolite_branch branch)

ami_id=$amiid
snapshot_id=$snapshotid

Test the new AMI using something like:

  instance_id=\$(aws ec2 run-instances \\
    --region $region \\
    --key \$USER \\
    --instance-type t1.micro \\
    --image-id $amiid
    --output text \\
    --query 'Instances[*].InstanceId' )
  echo instance_id=\$instance_id

EOF