Merge 3f2eb0a into 01b581b

taskcluster · Jul 18, 2019 · bf42c7d · bf42c7d
2 parents 01b581b + 3f2eb0a
commit bf42c7d
Show file tree

Hide file tree

Showing 86 changed files with 202 additions and 46 deletions.
diff --git a/.taskcluster.yml b/.taskcluster.yml
@@ -477,8 +477,8 @@ tasks:
   ############## Linux amd64 Multiuser Build ###############
   ##########################################################
 
-  - provisionerId: pmoore-manual
-    workerType: linux-amd64
+  - provisionerId: aws-provisioner-v1
+    workerType: gwci-linux
     metadata:
       name: "Build/test 64 bit generic-worker (multiuser engine) on Ubuntu 17.04 VM"
       description: "This builds the 64 bit linux version of generic-worker (multiuser engine)"
@@ -550,8 +550,8 @@ tasks:
   ################ Linux amd64 Simple Build ################
   ##########################################################
 
-  - provisionerId: pmoore-manual
-    workerType: linux-amd64
+  - provisionerId: aws-provisioner-v1
+    workerType: gwci-linux
     metadata:
       name: "Build/test 64 bit generic-worker (simple engine) on Ubuntu 17.04 VM"
       description: "This builds the 64 bit linux version of generic-worker (simple engine)"
@@ -621,8 +621,8 @@ tasks:
   ################ Linux amd64 Docker Build ################
   ##########################################################
 
-  - provisionerId: pmoore-manual
-    workerType: linux-amd64
+  - provisionerId: aws-provisioner-v1
+    workerType: gwci-linux
     metadata:
       name: "Build/test 64 bit generic-worker (docker engine) on Ubuntu 17.04 VM"
       description: "This builds the 64 bit linux version of generic-worker (docker engine)"

diff --git a/worker_types/ami-test-win2012r2/ami-base-name b/worker_types/ami-test-win2012r2/ami-base-name
diff --git a/worker_types/ami-test-win2012r2/filters b/worker_types/ami-test-win2012r2/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/ami-test-win2012r2/owners b/worker_types/ami-test-win2012r2/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/ami-test-win7sp1/ami-base-name b/worker_types/ami-test-win7sp1/ami-base-name
diff --git a/worker_types/ami-test-win7sp1/filters b/worker_types/ami-test-win7sp1/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-t-win7-32-base-*
diff --git a/worker_types/ami-test-win7sp1/owners b/worker_types/ami-test-win7sp1/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/deepspeech-win-b/ami-base-name b/worker_types/deepspeech-win-b/ami-base-name
diff --git a/worker_types/deepspeech-win-b/filters b/worker_types/deepspeech-win-b/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/deepspeech-win-b/owners b/worker_types/deepspeech-win-b/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/deepspeech-win-gpu-b/ami-base-name b/worker_types/deepspeech-win-gpu-b/ami-base-name
diff --git a/worker_types/deepspeech-win-gpu-b/filters b/worker_types/deepspeech-win-gpu-b/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/deepspeech-win-gpu-b/owners b/worker_types/deepspeech-win-gpu-b/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/deepspeech-win-gpu/ami-base-name b/worker_types/deepspeech-win-gpu/ami-base-name
diff --git a/worker_types/deepspeech-win-gpu/filters b/worker_types/deepspeech-win-gpu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/deepspeech-win-gpu/owners b/worker_types/deepspeech-win-gpu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/deepspeech-win/ami-base-name b/worker_types/deepspeech-win/ami-base-name
diff --git a/worker_types/deepspeech-win/filters b/worker_types/deepspeech-win/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/deepspeech-win/owners b/worker_types/deepspeech-win/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-1-b-win2012-beta/ami-base-name b/worker_types/gecko-1-b-win2012-beta/ami-base-name
diff --git a/worker_types/gecko-1-b-win2012-beta/filters b/worker_types/gecko-1-b-win2012-beta/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-1-b-win2012-beta/owners b/worker_types/gecko-1-b-win2012-beta/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-1-b-win2012/ami-base-name b/worker_types/gecko-1-b-win2012/ami-base-name
diff --git a/worker_types/gecko-1-b-win2012/filters b/worker_types/gecko-1-b-win2012/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-1-b-win2012/owners b/worker_types/gecko-1-b-win2012/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-2-b-win2012/ami-base-name b/worker_types/gecko-2-b-win2012/ami-base-name
diff --git a/worker_types/gecko-2-b-win2012/filters b/worker_types/gecko-2-b-win2012/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-2-b-win2012/owners b/worker_types/gecko-2-b-win2012/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-3-b-win2012/ami-base-name b/worker_types/gecko-3-b-win2012/ami-base-name
diff --git a/worker_types/gecko-3-b-win2012/filters b/worker_types/gecko-3-b-win2012/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-3-b-win2012/owners b/worker_types/gecko-3-b-win2012/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64-beta/ami-base-name b/worker_types/gecko-t-win10-64-beta/ami-base-name
diff --git a/worker_types/gecko-t-win10-64-beta/filters b/worker_types/gecko-t-win10-64-beta/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64-beta/owners b/worker_types/gecko-t-win10-64-beta/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64-cu/ami-base-name b/worker_types/gecko-t-win10-64-cu/ami-base-name
diff --git a/worker_types/gecko-t-win10-64-cu/filters b/worker_types/gecko-t-win10-64-cu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64-cu/owners b/worker_types/gecko-t-win10-64-cu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64-gpu-b/ami-base-name b/worker_types/gecko-t-win10-64-gpu-b/ami-base-name
diff --git a/worker_types/gecko-t-win10-64-gpu-b/filters b/worker_types/gecko-t-win10-64-gpu-b/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64-gpu-b/owners b/worker_types/gecko-t-win10-64-gpu-b/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64-gpu/ami-base-name b/worker_types/gecko-t-win10-64-gpu/ami-base-name
diff --git a/worker_types/gecko-t-win10-64-gpu/filters b/worker_types/gecko-t-win10-64-gpu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64-gpu/owners b/worker_types/gecko-t-win10-64-gpu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64-hw/ami-base-name b/worker_types/gecko-t-win10-64-hw/ami-base-name
diff --git a/worker_types/gecko-t-win10-64-hw/filters b/worker_types/gecko-t-win10-64-hw/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64-hw/owners b/worker_types/gecko-t-win10-64-hw/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win10-64/ami-base-name b/worker_types/gecko-t-win10-64/ami-base-name
diff --git a/worker_types/gecko-t-win10-64/filters b/worker_types/gecko-t-win10-64/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win10-64/owners b/worker_types/gecko-t-win10-64/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32-beta/ami-base-name b/worker_types/gecko-t-win7-32-beta/ami-base-name
diff --git a/worker_types/gecko-t-win7-32-beta/filters b/worker_types/gecko-t-win7-32-beta/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32-beta/owners b/worker_types/gecko-t-win7-32-beta/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32-cu/ami-base-name b/worker_types/gecko-t-win7-32-cu/ami-base-name
diff --git a/worker_types/gecko-t-win7-32-cu/filters b/worker_types/gecko-t-win7-32-cu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32-cu/owners b/worker_types/gecko-t-win7-32-cu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32-gpu-b/ami-base-name b/worker_types/gecko-t-win7-32-gpu-b/ami-base-name
diff --git a/worker_types/gecko-t-win7-32-gpu-b/filters b/worker_types/gecko-t-win7-32-gpu-b/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32-gpu-b/owners b/worker_types/gecko-t-win7-32-gpu-b/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32-gpu/ami-base-name b/worker_types/gecko-t-win7-32-gpu/ami-base-name
diff --git a/worker_types/gecko-t-win7-32-gpu/filters b/worker_types/gecko-t-win7-32-gpu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32-gpu/owners b/worker_types/gecko-t-win7-32-gpu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32-hw/ami-base-name b/worker_types/gecko-t-win7-32-hw/ami-base-name
diff --git a/worker_types/gecko-t-win7-32-hw/filters b/worker_types/gecko-t-win7-32-hw/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32-hw/owners b/worker_types/gecko-t-win7-32-hw/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gecko-t-win7-32/ami-base-name b/worker_types/gecko-t-win7-32/ami-base-name
diff --git a/worker_types/gecko-t-win7-32/filters b/worker_types/gecko-t-win7-32/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=gecko-b-win2012-base-*
diff --git a/worker_types/gecko-t-win7-32/owners b/worker_types/gecko-t-win7-32/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/gwci-linux/filters b/worker_types/gwci-linux/filters
@@ -0,0 +1 @@
+Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*
diff --git a/worker_types/gwci-linux/owners b/worker_types/gwci-linux/owners
@@ -0,0 +1 @@
+099720109477
diff --git a/worker_types/gwci-linux/userdata b/worker_types/gwci-linux/userdata
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# The output of this script gets logged to:
+#   /var/log/cloud-init-output.log
+
+set -exv
+
+exec &> /var/log/userdata.log
+
+function retry {
+  set +e
+  local n=0
+  local max=3
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:" >&2
+      else
+        echo "Failed after $n attempts." >&2
+        exit 1
+      fi
+    }
+  done
+  set -e
+}
+
+start_time="$(date '+%s')"
+
+# install docker
+retry apt update
+retry apt upgrade -y
+retry apt install -y apt-transport-https ca-certificates curl software-properties-common
+retry curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+retry apt update
+apt-cache policy docker-ce | grep -qF download.docker.com
+retry apt install -y docker-ce
+sleep 5
+systemctl status docker | grep "Started Docker Application Container Engine"
+usermod -aG docker ubuntu
+
+# install generic-worker into /home/ubuntu/generic-worker
+mkdir -p /home/ubuntu/generic-worker
+cd /home/ubuntu/generic-worker
+retry curl -L https://github.com/taskcluster/generic-worker/releases/download/v15.1.2/generic-worker-simple-linux-amd64 > generic-worker
+retry curl -L https://github.com/taskcluster/taskcluster-proxy/releases/download/v5.1.0/taskcluster-proxy-linux-amd64 > taskcluster-proxy
+retry curl -L https://github.com/taskcluster/livelog/releases/download/v1.1.0/livelog-linux-amd64 > livelog
+chmod a+x generic-worker taskcluster-proxy livelog
+chown -R ubuntu:ubuntu /home/ubuntu/generic-worker
+./generic-worker --version
+./generic-worker new-ed25519-keypair --file ed25519.key
+
+# ensure host 'taskcluster' resolves to localhost
+echo 127.0.1.1 taskcluster >> /etc/hosts
+
+# configure generic-worker to run on boot
+echo '@reboot PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin /home/ubuntu/generic-worker/generic-worker run --configure-for-aws --config /home/ubuntu/generic-worker/generic-worker.config >> /home/ubuntu/generic-worker/generic-worker.log 2>&1' | crontab -u ubuntu -
+
+end_time="$(date '+%s')"
+echo "UserData execution took: $(($end_time-$start_time)) seconds"
+
+# shutdown so that instance can be snapshotted
+shutdown now
diff --git a/worker_types/nss-win2012r2-new/ami-base-name b/worker_types/nss-win2012r2-new/ami-base-name
diff --git a/worker_types/nss-win2012r2-new/filters b/worker_types/nss-win2012r2-new/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/nss-win2012r2-new/owners b/worker_types/nss-win2012r2-new/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/nss-win2012r2/ami-base-name b/worker_types/nss-win2012r2/ami-base-name
diff --git a/worker_types/nss-win2012r2/filters b/worker_types/nss-win2012r2/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/nss-win2012r2/owners b/worker_types/nss-win2012r2/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/update.sh b/worker_types/update.sh
@@ -8,10 +8,9 @@ chmod 400 "${REGION}.id_rsa"
 
 # aws cli docs lie, they say userdata must be base64 encoded, but cli encodes for you, so just cat it...
 USER_DATA="$(cat userdata)"
-AMI_BASE_NAME="$(cat ami-base-name)"
 
-# find out latest windows 2012 r2 ami to use...
-AMI_METADATA="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=platform,Values=windows" "Name=name,Values=${AMI_BASE_NAME}" --query 'Images[*].{A:CreationDate,B:ImageId,C:Name}' --output text | sort -u | tail -1 | cut -f2,3)"
+# search for latest base AMI to use
+AMI_METADATA="$(aws --region "${REGION}" ec2 describe-images --owners $(cat owners) --filters $(cat filters) --query 'Images[*].{A:CreationDate,B:ImageId,C:Name}' --output text | sort -u | tail -1 | cut -f2,3)"
 
 AMI="$(echo $AMI_METADATA | sed 's/ .*//')"
 AMI_NAME="$(echo $AMI_METADATA | sed 's/.* //')"
@@ -39,28 +38,38 @@ log "I've triggered the creation of instance ${INSTANCE_ID} - it can take a \x1B
 aws --region "${REGION}" ec2 create-tags --resources "${INSTANCE_ID}" --tags "Key=WorkerType,Value=aws-provisioner-v1/${WORKER_TYPE}" "Key=Name,Value=${WORKER_TYPE} base instance" "Key=TC-Windows-Base,Value=true"
 log "I've tagged it with \"WorkerType\": \"aws-provisioner-v1/${WORKER_TYPE}\""
 
+sleep 1
+
 # grab public IP before it shuts down and loses it!
 PUBLIC_IP="$(aws --region "${REGION}" ec2 describe-instances --instance-id "${INSTANCE_ID}" --query 'Reservations[*].Instances[*].NetworkInterfaces[*].Association.PublicIp' --output text)"
 
+PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key ${REGION}.id_rsa --output text --query PasswordData)"
+
+log "To connect to the template instance (please don't do so until AMI creation process is completed"'!'"):"
+log ''
+
+if [ -n "${PASSWORD}" ]; then
+  # windows
+  log "             Public IP: ${PUBLIC_IP}"
+  log "             Username:  Administrator"
+  log "             Password:  ${PASSWORD}"
+else
+  # linux
+  log "             ssh -i '$(pwd)/${REGION}.id_rsa' ubuntu@${PUBLIC_IP}"
+fi
+
 # poll for a stopped state
 until aws --region "${REGION}" ec2 wait instance-stopped --instance-ids "${INSTANCE_ID}" >/dev/null 2>&1; do
-  log "  Waiting for instance ${INSTANCE_ID} to shut down..."
+  log "  Waiting for instance ${INSTANCE_ID} (IP ${PUBLIC_IP}) to shut down..."
   sleep 30
 done
 
 log "Now snapshotting the instance to create an AMI..."
 # now capture the AMI
-IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "firefox desktop builds on windows - taskcluster worker - version ${SLUGID}" --output text)"
+IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "generic-worker ${SLUGID}" --output text)"
 
 log "The AMI is currently being created: ${IMAGE_ID}"
 
-PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key ${REGION}.id_rsa --output text --query PasswordData)"
-
-log "To connect to the template instance (please don't do so until AMI creation process is completed"'!'"):"
-log ''
-log "             Public IP: ${PUBLIC_IP}"
-log "             Username:  Administrator"
-log "             Password:  ${PASSWORD}"
 log ''
 log "To monitor the AMI creation process, see:"
 log ''
@@ -78,7 +87,7 @@ touch "${REGION}.${IMAGE_ID}.latest-ami"
 {
     echo "Instance:  ${INSTANCE_ID}"
     echo "Public IP: ${PUBLIC_IP}"
-    echo "Password:  ${PASSWORD}"
+    [ -n "${PASSWORD}" ] && echo "Password:  ${PASSWORD}"
     echo "AMI:       ${IMAGE_ID}"
 } > "${REGION}.secrets"
 

diff --git a/worker_types/win2012r2-cu/ami-base-name b/worker_types/win2012r2-cu/ami-base-name
diff --git a/worker_types/win2012r2-cu/filters b/worker_types/win2012r2-cu/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/win2012r2-cu/owners b/worker_types/win2012r2-cu/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/win2012r2/ami-base-name b/worker_types/win2012r2/ami-base-name
diff --git a/worker_types/win2012r2/filters b/worker_types/win2012r2/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2012-R2_RTM-English-64Bit-Base*
diff --git a/worker_types/win2012r2/owners b/worker_types/win2012r2/owners
@@ -0,0 +1,2 @@
+self
+amazon
diff --git a/worker_types/win2016/ami-base-name b/worker_types/win2016/ami-base-name
diff --git a/worker_types/win2016/filters b/worker_types/win2016/filters
@@ -0,0 +1,2 @@
+Name=platform,Values=windows
+Name=name,Values=Windows_Server-2016-English-Full-Base-*
diff --git a/worker_types/win2016/owners b/worker_types/win2016/owners
@@ -0,0 +1,2 @@
+self
+aamazon