From 52587025b41ccad7cf43ee31a0ccc592b58dc6ec Mon Sep 17 00:00:00 2001
From: Pete Moore <pmoore@mozilla.com>
Date: Thu, 18 Jul 2019 07:41:23 +0200
Subject: [PATCH] Bug 1559777 - incorporated feedback from @milescrabill

---
 worker_types/gwci-linux/userdata | 73 +++++++++++++++++++++++---------
 worker_types/update.sh           | 26 ++++++------
 2 files changed, 68 insertions(+), 31 deletions(-)

diff --git a/worker_types/gwci-linux/userdata b/worker_types/gwci-linux/userdata
index f319f338..35cfc7e9 100644
--- a/worker_types/gwci-linux/userdata
+++ b/worker_types/gwci-linux/userdata
@@ -1,31 +1,66 @@
 #!/bin/bash
-mkdir -p /home/ubuntu/generic-worker
-cd /home/ubuntu/generic-worker
-apt update
-apt upgrade -y
-apt install -y apt-transport-https ca-certificates curl software-properties-common
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+set -exv
+exec &> /var/log/userdata.log
+
+# Version numbers ####################
+GENERIC_WORKER_VERSION='v15.1.2'
+LIVELOG_VERSION='v1.1.0'
+TASKCLUSTER_PROXY_VERSION='v5.1.0'
+######################################
+
+function retry {
+  set +e
+  local n=0
+  local max=3
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:" >&2
+      else
+        echo "Failed after $n attempts." >&2
+        exit 1
+      fi
+    }
+  done
+  set -e
+}
+
+start_time="$(date '+%s')"
+
+# install docker
+retry apt update
+retry apt upgrade -y
+retry apt install -y apt-transport-https ca-certificates curl software-properties-common
+retry curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
-apt update
+retry apt update
 apt-cache policy docker-ce | grep -qF download.docker.com
-apt install -y docker-ce
+retry apt install -y docker-ce
 sleep 5
 systemctl status docker | grep "Started Docker Application Container Engine"
 usermod -aG docker ubuntu
-id -nG | grep docker
-curl -L https://github.com/taskcluster/generic-worker/releases/download/v15.1.2/generic-worker-simple-linux-amd64 > generic-worker
-curl -L https://github.com/taskcluster/taskcluster-proxy/releases/download/v5.1.0/taskcluster-proxy-linux-amd64 > taskcluster-proxy
-curl -L https://github.com/taskcluster/livelog/releases/download/v1.1.0/livelog-linux-amd64 > livelog
+
+# install generic-worker into /home/ubuntu/generic-worker
+mkdir -p /home/ubuntu/generic-worker
+cd /home/ubuntu/generic-worker
+retry curl -L "https://github.com/taskcluster/generic-worker/releases/download/${GENERIC_WORKER_VERSION}/generic-worker-simple-linux-amd64" > generic-worker
+retry curl -L "https://github.com/taskcluster/livelog/releases/download/${LIVELOG_VERSION}/livelog-linux-amd64" > livelog
+retry curl -L "https://github.com/taskcluster/taskcluster-proxy/releases/download/${TASKCLUSTER_PROXY_VERSION}/taskcluster-proxy-linux-amd64" > taskcluster-proxy
 chmod a+x generic-worker taskcluster-proxy livelog
 chown -R ubuntu:ubuntu /home/ubuntu/generic-worker
 ./generic-worker --version
 ./generic-worker new-ed25519-keypair --file ed25519.key
+
+# ensure host 'taskcluster' resolves to localhost
+echo 127.0.1.1 taskcluster >> /etc/hosts
+
+# configure generic-worker to run on boot
 echo '@reboot PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin /home/ubuntu/generic-worker/generic-worker run --configure-for-aws --config /home/ubuntu/generic-worker/generic-worker.config >> /home/ubuntu/generic-worker/generic-worker.log 2>&1' | crontab -u ubuntu -
-# apt install -y gnome
-# sed -i 's/.*#.* \(AutomaticLoginEnable = true\)/\1/'
-# sed -i 's/.*#.* AutomaticLogin =.*/AutomaticLogin = ubuntu/'
-# mkdir -p .config/autostart
-# echo IyBUaGlzIFNvdXJjZSBDb2RlIEZvcm0gaXMgc3ViamVjdCB0byB0aGUgdGVybXMgb2YgdGhlIE1vemlsbGEgUHVibGljCiMgTGljZW5zZSwgdi4gMi4wLiBJZiBhIGNvcHkgb2YgdGhlIE1QTCB3YXMgbm90IGRpc3RyaWJ1dGVkIHdpdGggdGhpcwojIGZpbGUsIFlvdSBjYW4gb2J0YWluIG9uZSBhdCBodHRwOi8vbW96aWxsYS5vcmcvTVBMLzIuMC8uCltEZXNrdG9wIEVudHJ5XQpUeXBlPUFwcGxpY2F0aW9uCkV4ZWM9Z25vbWUtdGVybWluYWwgLXggL2hvbWUvdWJ1bnR1L2dlbmVyaWMtd29ya2VyIHJ1biAtLWNvbmZpZyAvaG9tZS91YnVudHUvZ2VuZXJpYy13b3JrZXIuY29uZmlnIC0tY29uZmlndXJlLWZvci1hd3MgPiAvaG9tZS91YnVudHUvZ2VuZXJpYy13b3JrZXIubG9nIDI+JjEKSGlkZGVuPWZhbHNlClgtR05PTUUtQXV0b3N0YXJ0LWVuYWJsZWQ9dHJ1ZQpOYW1lPUdlbmVyaWMtV29ya2VyCkNvbW1lbnQ9U3RhcnQgZ2VuZXJpYy13b3JrZXIgaW4gYSB0ZXJtaW5hbCBzZXNzaW9uClN0YXJ0dXBOb3RpZnk9ZmFsc2UKVGVybWluYWw9ZmFsc2UKVHlwZT1BcHBsaWNhdGlvbgo= | base64 -d > .config/autostart/gnome-terminal.desktop
-# apt install -y xrdp
-# passwd ubuntu
+
+end_time="$(date '+%s')"
+echo "UserData execution took: $(($end_time-$start_time)) seconds"
+
+# shutdown so that instance can be snapshotted
 shutdown now
diff --git a/worker_types/update.sh b/worker_types/update.sh
index 6e715ed5..dc1f7c60 100755
--- a/worker_types/update.sh
+++ b/worker_types/update.sh
@@ -38,21 +38,11 @@ log "I've triggered the creation of instance ${INSTANCE_ID} - it can take a \x1B
 aws --region "${REGION}" ec2 create-tags --resources "${INSTANCE_ID}" --tags "Key=WorkerType,Value=aws-provisioner-v1/${WORKER_TYPE}" "Key=Name,Value=${WORKER_TYPE} base instance" "Key=TC-Windows-Base,Value=true"
 log "I've tagged it with \"WorkerType\": \"aws-provisioner-v1/${WORKER_TYPE}\""
 
+sleep 1
+
 # grab public IP before it shuts down and loses it!
 PUBLIC_IP="$(aws --region "${REGION}" ec2 describe-instances --instance-id "${INSTANCE_ID}" --query 'Reservations[*].Instances[*].NetworkInterfaces[*].Association.PublicIp' --output text)"
 
-# poll for a stopped state
-until aws --region "${REGION}" ec2 wait instance-stopped --instance-ids "${INSTANCE_ID}" >/dev/null 2>&1; do
-  log "  Waiting for instance ${INSTANCE_ID} (IP ${PUBLIC_IP}) to shut down..."
-  sleep 30
-done
-
-log "Now snapshotting the instance to create an AMI..."
-# now capture the AMI
-IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "firefox desktop builds on windows - taskcluster worker - version ${SLUGID}" --output text)"
-
-log "The AMI is currently being created: ${IMAGE_ID}"
-
 PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key ${REGION}.id_rsa --output text --query PasswordData)"
 
 log "To connect to the template instance (please don't do so until AMI creation process is completed"'!'"):"
@@ -68,6 +58,18 @@ else
   log "             ssh -i '$(pwd)/${REGION}.id_rsa' ubuntu@${PUBLIC_IP}"
 fi
 
+# poll for a stopped state
+until aws --region "${REGION}" ec2 wait instance-stopped --instance-ids "${INSTANCE_ID}" >/dev/null 2>&1; do
+  log "  Waiting for instance ${INSTANCE_ID} (IP ${PUBLIC_IP}) to shut down..."
+  sleep 30
+done
+
+log "Now snapshotting the instance to create an AMI..."
+# now capture the AMI
+IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "generic-worker ${SLUGID}" --output text)"
+
+log "The AMI is currently being created: ${IMAGE_ID}"
+
 log ''
 log "To monitor the AMI creation process, see:"
 log ''