Add cloudbuild.yaml deployments for ePoxy container on GCE

Dockerfile

@@ -0,0 +1,19 @@
+FROM golang:1.11 as build
+
+# Add the local files to be sure we are building the local source code instead
+# of downloading from GitHub. All other package dependencies will be downloaded
+# from HEAD.
+ADD . /go/src/github.com/m-lab/epoxy
+RUN CGO_ENABLED=0 go get -v github.com/m-lab/epoxy/cmd/epoxy_boot_server
+
+# Now copy the built binary into a minimal base image.
+FROM alpine
+COPY --from=build /go/bin/epoxy_boot_server /
+
+# We must install the ca-certificates package so the ePoxy server can securely
+# connect to the LetsEncrypt servers to register & create our certificates.
+# As well, valid ca-certificates are needed for the storage proxy connections.
+RUN apk update && apk add ca-certificates
+
+WORKDIR /
+ENTRYPOINT ["/epoxy_boot_server"]

README.md

@@ -1,15 +1,39 @@
 # ePoxy
+
 A system for safe boot management over the Internet, based on iPXE.
 
-# Building
+## Building
 
 To build the ePoxy boot server:
 
     go get github.com/m-lab/epoxy/cmd/epoxy_boot_server
 
-# Testing
+## Deployment
+
+The ePoxy server is designed to run from within a docker container. The M-Lab
+deployment targets a stand-alone GCE VM. The cloudbuild.yaml configuration
+embeds static zones for specific regional deployments for each GCP project.
+
+Before deploying to a new Project complete the following steps in advance:
+
+* Allocate static IP address and register DNS
+
+      PROJECT=mlab-sandbox ZONE=us-east1-c setup_epoxy_dns.sh
+
+* Allocate server certificte and key
+
+      TODO: add steps to allocate server certs.
+
+* Create GCS bucket `gs://epoxy-${PROJECT}-private` and copy server certificate
+  & key.
+
+      gsutil mb -p mlab-sandbox gs://epoxy-mlab-sandbox-private
+      gsutil cp server-certs.pem server-key.pem gs://epoxy-mlab-sandbox-private
+
+
+## Testing
 
-## Testing Server
+### Testing Server
 
 The datastore emulator depends on the [Google Cloud
 SDK](https://cloud.google.com/sdk/downloads). After installing `gcloud`,
@@ -38,7 +62,7 @@ Start the epoxy server:
 The ePoxy server is now connected to the local datastore emulator, and can
 serve client requests.
 
-## Testing Client
+### Testing Client
 
 After starting the datastore emuulator and a local epoxy boot server, you can
 simulate a client request using `curl`.

cloudbuild.yaml

@@ -0,0 +1,27 @@
+# Timeout for complete build. Default is 10m.
+timeout: 1800s
+
+steps:
+# Create the image for later steps.
+- name: gcr.io/cloud-builders/docker
+  args: [
+    'build', '-t', 'gcr.io/$PROJECT_ID/epoxy_boot_server:$BUILD_ID', '.'
+  ]
+# Make the new container available immediately.
+- name: gcr.io/cloud-builders/docker
+  args: [
+    'push', 'gcr.io/$PROJECT_ID/epoxy_boot_server:$BUILD_ID'
+  ]
+# Deploy to GCE.
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: bash
+  args:
+   - '/workspace/deploy_epoxy_container.sh'
+  env:
+   - 'PROJECT=$PROJECT_ID'
+   - 'CONTAINER=gcr.io/$PROJECT_ID/epoxy_boot_server:$BUILD_ID'
+   # NOTE: Changing zones will require manual intervention.
+   - 'ZONE_mlab_sandbox=us-east1-c'
+   - 'ZONE_mlab_staging=us-central1-b'
+   - 'ZONE_mlab_oti=us-east1-c'
+

deploy_epoxy_container.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# deploy_epoxy_container.sh creates a GCE VM, which runs a startup script, and
+# starts a container image for the epoxy_boot_server. Because the epoxy boot api
+# uses a static IP, that IP is unassigned from the current VM and reassigned to
+# the new VM, once the new VM appears to be working.
+#
+# deploy_epoxy_container.sh depends on three environment variables for correct
+# operation.
+#
+#  PROJECT - specifies the GCP project name to create the VM, e.g. mlab-sandbox.
+#  CONTAINER - specifies the docker container image URL, e.g.
+#                gcr.io/mlab-sandbox/epoxy_boot_server
+#  ZONE_<project> - specifies the GCP VM zone, e.g. ZONE_mlab_sandbox=us-east1-c
+#
+# Example usage:
+#
+#   PROJECT=mlab-sandbox \
+#   CONTAINER=gcr.io/mlab-sandbox/epoxy_boot_server:$BUILD_ID \
+#   ZONE_mlab_sandbox=us-east1-c \
+#     ./deploy_epoxy_container.sh
+
+set -ex
+
+zone_ref=ZONE_${PROJECT//-/_}
+ZONE=${!zone_ref}
+
+if [[ -z "${PROJECT}" || -z "${CONTAINER}" || -z "${ZONE}" ]]; then
+  echo "ERROR: PROJECT, CONTAINER, and ZONE must be defined in environment."
+  echo "ERROR: Current values are:"
+  echo "  PROJECT='${PROJECT}'"
+  echo "  CONTAINER='${CONTAINER}'"
+  echo "  ZONE='${ZONE}'"
+  exit 1
+fi
+
+# Lookup address.
+IP=$(gcloud compute addresses describe --project "${PROJECT}" \
+  --format "value(address)" --region "${ZONE%-*}" epoxy-boot-api)
+if [[ -z "${IP}" ]]; then
+  echo "ERROR: Failed to find static IP in region ${ZONE%-*}"
+  echo "ERROR: Run the m-lab/epoxy/setup_epoxy_dns.sh to allocate one."
+  exit 1
+fi
+# Lookup the instance (if any) currently using the static IP address for ePoxy.
+gce_url=$(gcloud compute addresses describe --project "${PROJECT}" \
+  --format "value(users)" --region "${ZONE%-*}" epoxy-boot-api)
+CURRENT_INSTANCE=${gce_url##*/}
+UPDATED_INSTANCE="epoxy-boot-api-$(date +%Y%m%dt%H%M%S)"
+
+CERTDIR=/home/epoxy
+# Create startup script to pass to create instance. Script will run as root.
+cat <<EOF >startup.sh
+#!/bin/bash
+set -x
+mkdir "${CERTDIR}"
+# Copy certificates from GCS.
+# Retry because docker fails to contact gcr.io sometimes.
+until docker run --tty --volume "${CERTDIR}:${CERTDIR}" \
+  gcr.io/cloud-builders/gsutil \
+  cp gs://epoxy-${PROJECT}-private/server-certs.pem \
+      gs://epoxy-${PROJECT}-private/server-key.pem \
+      "${CERTDIR}"; do
+  sleep 5
+done
+EOF
+
+cat <<EOF >config.env
+IPXE_CERT_FILE=/certs/server-certs.pem
+IPXE_KEY_FILE=/certs/server-key.pem
+PUBLIC_HOSTNAME=epoxy-boot-api.${PROJECT}.measurementlab.net
+STORAGE_PREFIX_URL=https://storage.googleapis.com/epoxy-${PROJECT}
+GCLOUD_PROJECT=${PROJECT}
+EOF
+
+# Create new VM without public IP.
+gcloud compute instances create-with-container "${UPDATED_INSTANCE}" \
+  --project "${PROJECT}" \
+  --zone "${ZONE}" \
+  --tags allow-epoxy-ports \
+  --scopes default,datastore \
+  --metadata-from-file "startup-script=startup.sh" \
+  --network-interface network=mlab-platform-network,subnet=epoxy \
+  --container-image "${CONTAINER}" \
+  --container-mount-host-path host-path=/home/epoxy,mount-path=/certs \
+  --container-env-file config.env
+
+sleep 20
+TEMP_IP=$(gcloud compute instances describe \
+  --project "${PROJECT}" --zone "${ZONE}" \
+  --format 'value(networkInterfaces[].accessConfigs[0].natIP)' \
+  ${UPDATED_INSTANCE})
+
+# Run a basic diagnostic test.
+while ! curl --insecure --dump-header - https://${TEMP_IP}:4430/_ah/health; do
+  sleep 5
+done
+
+# Remove public IP from updated instance so we can assign the (now available)
+# static IP.
+gcloud compute instances delete-access-config --zone "${ZONE}" \
+  --project "${PROJECT}" \
+  --access-config-name "external-nat" "${UPDATED_INSTANCE}"
+
+if [[ -n "${CURRENT_INSTANCE}" ]]; then
+  # Remove public IP from current instance so we can assign it to the new one.
+  gcloud compute instances delete-access-config --zone "${ZONE}" \
+    --project "${PROJECT}" \
+    --access-config-name "external-nat" "${CURRENT_INSTANCE}"
+fi
+
+# Assign the static IP to the updated instance.
+gcloud compute instances add-access-config --zone "${ZONE}" \
+  --project "${PROJECT}" \
+  --access-config-name "external-nat" --address "$IP" \
+  "${UPDATED_INSTANCE}"

setup_epoxy_dns.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# setup_epoxy_dns.sh looks up the current epoxy-boot-api static IP. If one is
+# not found, a new one is created. Then, setup_epoxy_dns.sh looks up the
+# current DNS record for epoxy-boot-api.<project>.measurementlab.net and
+# updates it if needed.
+#
+# setup_epoxy_dns.sh should be safe to run multiple times. Typically, this is
+# only needed once per project.
+#
+# EXAMPLE USAGE:
+#   PROJECT=mlab-sandbox ZONE=us-east1-c ./setup_epoxy_dns.sh
+
+if [[ -z "${PROJECT}" || -z "${ZONE}" ]] ; then
+    echo "ERROR: Both PROJECT= and ZONE= must be set in the environment"
+    echo "ERROR: e.g. PROJECT=mlab-sandbox ZONE=us-east1-c"
+    exit 1
+fi
+
+set -xe
+IP=$( gcloud compute addresses describe --project "${PROJECT}" \
+        --format "value(address)" --region "${ZONE%-*}" epoxy-boot-api || : )
+if [[ -z "${IP}" ]] ; then
+    gcloud compute addresses create epoxy-boot-api \
+        --project "${PROJECT}" \
+        --region "${ZONE%-*}"
+    IP=$( gcloud compute addresses describe --project "${PROJECT}" \
+            --format "value(address)" --region "${ZONE%-*}" epoxy-boot-api )
+fi
+
+CURRENT_IP=$(
+    gcloud dns record-sets list --zone "${PROJECT}-measurementlab-net" \
+       --name "epoxy-boot-api.${PROJECT}.measurementlab.net." \
+       --format "value(rrdatas[0])" --project "${PROJECT}" )
+if [[ "${CURRENT_IP}" != "${IP}" ]] ; then
+
+    # Add the record, deleting the existing one first.
+    gcloud dns record-sets transaction start \
+        --zone "${PROJECT}-measurementlab-net" \
+        --project "${PROJECT}"
+	# Allow remove to fail when CURRENT_IP is empty.
+    gcloud dns record-sets transaction remove \
+        --zone "${PROJECT}-measurementlab-net" \
+        --name "epoxy-boot-api.${PROJECT}.measurementlab.net." \
+        --type A \
+        --ttl 300 \
+        "${CURRENT_IP}" \
+        --project "${PROJECT}" || :
+    gcloud dns record-sets transaction add \
+        --zone "${PROJECT}-measurementlab-net" \
+        --name "epoxy-boot-api.${PROJECT}.measurementlab.net." \
+        --type A \
+        --ttl 300 \
+        "${IP}" \
+        --project "${PROJECT}"
+    gcloud dns record-sets transaction execute \
+        --zone "${PROJECT}-measurementlab-net" \
+        --project "${PROJECT}"
+
+fi