Merge pull request #49 from alphagov/add-disk-attach

Ensure that an EC2 instance will die if not attached to an EBS volume
alphagov · Jun 29, 2018 · cd24320 · cd24320
2 parents 8427123 + ad947d2
commit cd24320
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 11 deletions.
diff --git a/terraform/projects/app-ecs-instances/ecs-iam-policy.tf b/terraform/projects/app-ecs-instances/ecs-iam-policy.tf
@@ -66,6 +66,7 @@ data "aws_iam_policy_document" "ecs_instance_document" {
 
     actions = [
       "ec2:DescribeVolumes",
+      "ec2:DescribeVolumeStatus",
     ]
   }
 }

diff --git a/terraform/projects/app-ecs-instances/instance-user-data.tpl b/terraform/projects/app-ecs-instances/instance-user-data.tpl
@@ -1,21 +1,40 @@
 #!/bin/bash
 # Attach EBS volume to instance
 echo "[$(date '+%H:%M:%S %d-%m-%Y')] installing dependencies for volume attaching"
-sudo yum install -y aws-cli wget
 
 REGION="${region}"
 DEVICE="xvdf"
-VOLUME_IDS="${volume_ids}"
 
 echo "[$(date '+%H:%M:%S %d-%m-%Y')] finding current instance ID"
-INSTANCE_ID="`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`"
+INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
 
 echo "[$(date '+%H:%M:%S %d-%m-%Y')] finding volume to attach"
 AZ="$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone)"
-VOLUME_ID="$(aws ec2 describe-volumes --filters Name=availability-zone,Values=$AZ --volume-ids $VOLUME_IDS --region $REGION --query Volumes[*].VolumeId --output text)"
+VOLUME_ID="$(aws ec2 describe-volumes --filters Name=availability-zone,Values="$AZ" --volume-ids ${volume_ids} --region "$REGION" --query Volumes[*].VolumeId --output text)"
 
-echo "[$(date '+%H:%M:%S %d-%m-%Y')] attaching volume: $VOLUME_ID"
-aws ec2 attach-volume --volume-id $VOLUME_ID --instance-id $INSTANCE_ID --device /dev/$DEVICE --region $REGION
+count=0
+DISK_AVAILABILITY="NA"
+until [ "$DISK_AVAILABILITY" = available ]; do
+    if [[ $count -le 10 ]]
+    then
+        sleep 10;
+        echo "Sleeping: waiting for volume to become available"
+        count=$((count+1));
+        DISK_AVAILABILITY=$(aws ec2 describe-volumes --region "$REGION" --filters Name=volume-id,Values="$VOLUME_ID" --query Volumes[0].State --output text)
+    else
+        break
+    fi
+done
+
+case $DISK_AVAILABILITY in
+        available)
+            echo "[$(date '+%H:%M:%S %d-%m-%Y')] attaching volume: $VOLUME_ID"
+            aws ec2 attach-volume --volume-id "$VOLUME_ID" --instance-id "$INSTANCE_ID" --device /dev/"$DEVICE" --region "$REGION";
+         ;;
+        *)
+            shutdown -h now;
+         ;;
+esac
 
 # Waiting for volume to finish attaching
 x=0
@@ -33,17 +52,16 @@ if file -s /dev/$DEVICE | grep -q "/dev/$DEVICE: data"; then
   echo "[$(date '+%H:%M:%S %d-%m-%Y')] attach-volume: /dev/$DEVICE does not contain any partition, beginning to format disk"
   mkfs -t ext4 /dev/$DEVICE
 else
-  echo "[$(date '+%H:%M:%S %d-%m-%Y')] attach-volume: /dev/$DEVICE is already formatted: $(file -s /dev/$DEVICE)"
+  echo "[$(date '+%H:%M:%S %d-%m-%Y')] attach-volume: /dev/$DEVICE is already formatted: $(file -s /dev/"$DEVICE")"
 fi
 
-
 #Mount volume to be used by prometheus container
 mkdir -p /ecs/prometheus_data
-mount /dev/$DEVICE /ecs/prometheus_data
+mount /dev/"$DEVICE" /ecs/prometheus_data
 
 
 
-#Create prometheus group and allow it to read and write to our volume for storing prometheus data. Note, 65534 is 
+#Create prometheus group and allow it to read and write to our volume for storing prometheus data. Note, 65534 is
 #chosen as the UID to be added to the prometheus group as this is the UID that prometheus in the docker container runs as.
 
 groupadd --system --gid 65534 prometheus
@@ -52,7 +70,7 @@ chown prometheus:prometheus /ecs/prometheus_data
 chmod -R 760 /ecs/prometheus_data
 
 # Set any ECS agent configuration options
-echo 'ECS_CLUSTER=${cluster_name}' >> /etc/ecs/ecs.config
+echo "ECS_CLUSTER=${cluster_name}" >> /etc/ecs/ecs.config
 yum install -y ecs-init
 start ecs
 service docker start