[WIP][NOT READY] Replace references to AWS

cloud/services/compute/instances.go

@@ -20,7 +20,6 @@ import (
 	"encoding/base64"
 	"fmt"
 
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/blang/semver"
 	"github.com/pkg/errors"
 	"google.golang.org/api/compute/v1"
@@ -125,7 +124,7 @@ func (s *Service) CreateInstance(scope *scope.MachineScope) (*compute.Instance,
 	input.Labels = infrav1.Build(infrav1.BuildParams{
 		ClusterName: s.scope.Name(),
 		Lifecycle:   infrav1.ResourceLifecycleOwned,
-		Role:        aws.String(scope.Role()),
+		Role:        pointer.StringPtr(scope.Role()),
 		// TODO(vincepri): Check what needs to be added for the cloud provider label.
 		Additional: s.scope.
 			GCPCluster.Spec.

examples/generate.sh

@@ -21,8 +21,9 @@ SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 OUTPUT_DIR=${OUTPUT_DIR:-${SOURCE_DIR}/_out}
 
 # Binaries
-ENVSUBST=${ENVSUBST:-envsubst}
-command -v "${ENVSUBST}" >/dev/null 2>&1 || echo -v "Cannot find ${ENVSUBST} in path."
+envsubst() {
+  python -c 'import os,sys;[sys.stdout.write(os.path.expandvars(l)) for l in sys.stdin]'
+}
 
 # Cluster.
 export CLUSTER_NAME="${CLUSTER_NAME:-test1}"

go.mod

@@ -3,7 +3,6 @@ module sigs.k8s.io/cluster-api-provider-gcp
 go 1.12
 
 require (
-	github.com/aws/aws-sdk-go v1.20.19
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/go-logr/logr v0.1.0
 	github.com/onsi/ginkgo v1.8.0

go.sum

@@ -21,8 +21,6 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/aws/aws-sdk-go v1.20.19 h1:RQDLGGlcffQzAceEXGdMu+uGGPGhNu+vNG3BrUZAMPI=
-github.com/aws/aws-sdk-go v1.20.19/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=

hack/checkout_account.py

@@ -24,7 +24,7 @@
 
 BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos")
 
-RESOURCE_TYPE = "gcp-account"
+RESOURCE_TYPE = "gce-project"
 USER = "cluster-api-provider-gcp"
 
 def post_request(host, input_state = "clean"):

hack/ci/e2e-conformance.sh

@@ -25,47 +25,143 @@ CLUSTER_NAME=${CLUSTER_NAME:-"test1"}
 
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 
-# our exit handler (trap)
-cleanup() {
+# dump logs from kind and all the nodes
+dump-logs() {
   # always attempt to dump logs
   kind "export" logs --name="clusterapi" "${ARTIFACTS}/logs" || true
+
+  # iterate through any nodes we brought up and collect logs
+  gcloud compute instances list --project "${GCP_PROJECT}" --format='value(name, zone)' \
+  | grep "${CLUSTER_NAME}" | while read node_name node_zone; do
+    echo "collecting logs from ${node_name}"
+    dir="${ARTIFACTS}/logs/${node_name}"
+    mkdir -p ${dir}
+
+    gcloud compute instances get-serial-port-output --project "${GCP_PROJECT}" \
+      --zone "${node_zone}" --port 1 "${node_name}" > "${dir}/serial-1.log" || true
+
+    ssh-to-node "${node_name}" "${node_zone}" "sudo chmod -R a+r /var/log" || true
+    gcloud compute scp --recurse --project "${GCP_PROJECT}" --zone "${node_zone}" \
+      "${node_name}:/var/log/cloud-init.log" "${node_name}:/var/log/cloud-init-output.log" "${dir}" || true
+
+    ssh-to-node "${node_name}" "${node_zone}" "sudo journalctl --output=short-precise" > "${dir}/systemd.log" || true
+  done
+
+  timeout 120 gcloud logging read --order=asc \
+    --freshness="3h" \
+    --format='table(timestamp,jsonPayload.resource.name,jsonPayload.event_subtype)' \
+    --project "${GCP_PROJECT}" > "${ARTIFACTS}/logs/activity.log" || true
+}
+
+# our exit handler (trap)
+cleanup() {
+  # dump all the logs
+  dump-logs
+
   # KIND_IS_UP is true once we: kind create
   if [[ "${KIND_IS_UP:-}" = true ]]; then
-    kubectl \
+    timeout 60 kubectl \
       --kubeconfig=$(kind get kubeconfig-path --name="clusterapi") \
       delete cluster test1 || true
-    kubectl \
+     timeout 60 kubectl \
       --kubeconfig=$(kind get kubeconfig-path --name="clusterapi") \
       wait --for=delete cluster/test1 || true
     make kind-reset || true
   fi
   # clean up e2e.test symlink
   (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true
+
+  # Force a cleanup of cluster api created resources using gcloud commands
+  gcloud compute forwarding-rules delete --project $GCP_PROJECT --global $CLUSTER_NAME-apiserver --quiet || true
+  gcloud compute target-tcp-proxies delete --project $GCP_PROJECT $CLUSTER_NAME-apiserver --quiet || true
+  gcloud compute backend-services delete --project $GCP_PROJECT --global $CLUSTER_NAME-apiserver --quiet || true
+  gcloud compute health-checks delete --project $GCP_PROJECT $CLUSTER_NAME-apiserver --quiet || true
+  (gcloud compute instances list --project $GCP_PROJECT | grep $CLUSTER_NAME \
+       | awk '{print "gcloud compute instances delete --project '$GCP_PROJECT' --quiet " $1 " --zone " $2 "\n"}' \
+       | bash) || true
+  (gcloud compute instance-groups list --project $GCP_PROJECT | grep $CLUSTER_NAME \
+       | awk '{print "gcloud compute instance-groups unmanaged delete --project '$GCP_PROJECT' --quiet " $1 " --zone " $2 "\n"}' \
+       | bash) || true
+  (gcloud compute firewall-rules list --project $GCP_PROJECT | grep $CLUSTER_NAME \
+       | awk '{print "gcloud compute firewall-rules delete --project '$GCP_PROJECT' --quiet " $1 "\n"}' \
+       | bash) || true
+
+  # cleanup the networks
+  gcloud compute routers nats delete "${CLUSTER_NAME}-mynat" --router-region="${GCP_REGION}" \
+    --router="${CLUSTER_NAME}-myrouter" --quiet || true
+  gcloud compute routers delete "${CLUSTER_NAME}-myrouter" --project="${GCP_PROJECT}" \
+    --region="${GCP_REGION}" --quiet || true
+
   # remove our tempdir
   # NOTE: this needs to be last, or it will prevent kind delete
   if [[ -n "${TMP_DIR:-}" ]]; then
-    rm -rf "${TMP_DIR}"
+    rm -rf "${TMP_DIR}" || true
   fi
 }
 
+# SSH to a node by name ($1) and run a command ($2).
+function ssh-to-node() {
+  local node="$1"
+  local zone="$2"
+  local cmd="$3"
+
+  # ensure we have an IP to connect to
+  gcloud compute --project "${GCP_PROJECT}" instances add-access-config "${node}" || true
+
+  # Loop until we can successfully ssh into the box
+  for try in {1..5}; do
+    if gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" \
+      --project "${GCP_PROJECT}" --zone "${zone}" "${node}" --command "echo test > /dev/null"; then
+      break
+    fi
+    sleep 5
+  done
+  # Then actually try the command.
+  gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" \
+    --project "${GCP_PROJECT}" --zone "${zone}" "${node}" --command "${cmd}"
+}
+
 init_image() {
   image=$(gcloud compute images list --project $GCP_PROJECT \
     --no-standard-images --filter="family:capi-ubuntu-1804-k8s" --format="table[no-heading](name)")
   if [[ -z "$image" ]]; then
+      if ! command -v ansible &> /dev/null; then
+        if [[ $EUID -ne 0 ]]; then
+          echo "Please install ansible and try again."
+          exit 1
+        else
+          # we need pip to install ansible
+          curl -L https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+          python get-pip.py --user
+          rm -f get-pip.py
+
+          # install ansible needed by packer
+          version="2.8.5"
+          python -m pip install "ansible==${version}"
+        fi
+      fi
       if ! command -v packer &> /dev/null; then
         hostos=$(go env GOHOSTOS)
         hostarch=$(go env GOHOSTARCH)
         version="1.4.3"
         url="https://releases.hashicorp.com/packer/${version}/packer_${version}_${hostos}_${hostarch}.zip"
         echo "Downloading packer from $url"
-        wget -O packer.zip $url  && \
+        wget --quiet -O packer.zip $url  && \
           unzip packer.zip && \
           rm packer.zip && \
-          mv packer "$(go env GOPATH)/bin"
+          ln -s $PWD/packer /usr/local/bin/packer
+      fi
+      if [[ $EUID -ne 0 ]]; then
+        (cd "$(go env GOPATH)/src/sigs.k8s.io/image-builder/images/capi" && \
+          GCP_PROJECT_ID=$GCP_PROJECT GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
+          make build-gce-default)
+      else
+        # assume we are running in the CI environment as root
+        # Add a user for ansible to work properly
+        groupadd -r packer && useradd -m -s /bin/bash -r -g packer packer
+        # use the packer user to run the build
+        su - packer -c "bash -c 'cd /go/src/sigs.k8s.io/image-builder/images/capi && GCP_PROJECT_ID=$GCP_PROJECT GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS make build-gce-default'"
       fi
-      (cd "$(go env GOPATH)/src/sigs.k8s.io/image-builder/images/capi" && \
-      GCP_PROJECT_ID=$GCP_PROJECT GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
-      make build-gce-default)
   fi
 }
 
@@ -96,6 +192,9 @@ build() {
 
 # generate manifests needed for creating the GCP cluster to run the tests
 generate_manifests() {
+  if ! command -v kustomize >/dev/null 2>&1; then
+    GO111MODULE=on go install sigs.k8s.io/kustomize/v3/cmd/kustomize
+  fi
 GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
 	GCP_REGION=$GCP_REGION \
 	GCP_PROJECT=$GCP_PROJECT \
@@ -109,16 +208,29 @@ create_cluster() {
   KIND_IS_UP=true
   make create-cluster
 
-  # Wait till all machines are running
+  # Wait till all machines are running (bail out at 30 mins)
+  attempt=0
   while true; do
+    kubectl get machines --kubeconfig=$(kind get kubeconfig-path --name="clusterapi")
     read running total <<< $(kubectl get machines --kubeconfig=$(kind get kubeconfig-path --name="clusterapi") \
-     -o json | jq -r '.items[].status.phase' | awk '/running/ {count++} END{print count " " NR}') ;
+      -o json | jq -r '.items[].status.phase' | awk 'BEGIN{count=0} /running/{count++} END{print count " " NR}') ;
     if [[ $total == "5" && $running == "5" ]]; then
-      return
+      return 0
+    fi
+    read failed total <<< $(kubectl get machines --kubeconfig=$(kind get kubeconfig-path --name="clusterapi") \
+      -o json | jq -r '.items[].status.phase' | awk 'BEGIN{count=0} /failed/{count++} END{print count " " NR}') ;
+    if [[ ! $failed -eq 0 ]]; then
+      echo "$failed machines (out of $total) in cluster failed ... bailing out"
+      exit 1
     fi
     timestamp=$(date +"[%H:%M:%S]")
+    if [ $attempt -gt 180 ]; then
+      echo "cluster did not start in 30 mins ... bailing out!"
+      exit 1
+    fi
     echo "$timestamp Total machines : $total / Running : $running .. waiting for 10 seconds"
     sleep 10
+    attempt=$((attempt+1))
   done
 }
 
@@ -148,7 +260,7 @@ run_tests() {
     | grep -cv "node-role.kubernetes.io/master" )"
 
   # wait for all the nodes to be ready
-  kubectl wait --for=condition=Ready node --kubeconfig=$KUBECONFIG --all
+  kubectl wait --for=condition=Ready node --kubeconfig=$KUBECONFIG --all || true
 
   # setting this env prevents ginkg e2e from trying to run provider setup
   export KUBERNETES_CONFORMANCE_TEST="y"
@@ -162,6 +274,16 @@ run_tests() {
   unset KUBERNETES_CONFORMANCE_TEST
 }
 
+# initialize a router and cloud NAT
+init_networks() {
+  gcloud version
+  gcloud compute routers create "${CLUSTER_NAME}-myrouter" --project="${GCP_PROJECT}" \
+    --region="${GCP_REGION}" --network=default
+  gcloud compute routers nats create "${CLUSTER_NAME}-mynat" --project="${GCP_PROJECT}" \
+    --router-region="${GCP_REGION}" --router="${CLUSTER_NAME}-myrouter" \
+    --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
+}
+
 # setup kind, build kubernetes, create a cluster, run the e2es
 main() {
   if [[ -z "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then
@@ -199,6 +321,7 @@ EOF
   source "${REPO_ROOT}/hack/ensure-kind.sh"
 
   # now build and run the cluster and tests
+  init_networks
   init_image
   build
   generate_manifests

main.go

@@ -92,13 +92,13 @@ func main() {
 	flag.IntVar(&gcpClusterConcurrency,
 		"gcpcluster-concurrency",
 		10,
-		"Number of AWSClusters to process simultaneously",
+		"Number of GCPClusters to process simultaneously",
 	)
 
 	flag.IntVar(&gcpMachineConcurrency,
 		"gcpmachine-concurrency",
 		10,
-		"Number of AWSMachines to process simultaneously",
+		"Number of GCPMachines to process simultaneously",
 	)
 
 	flag.DurationVar(&syncPeriod,

scripts/ci-e2e.sh

@@ -22,38 +22,35 @@
 set -o nounset
 set -o pipefail
 
-# our exit handler (trap)
-cleanup() {
-  # If Boskos is being used then release the GCP project back to Boskos.
-  [ -z "${BOSKOS_HOST:-}" ] || hack/checkin_account.py
-}
-
-trap cleanup EXIT
+BOSKOS_HOST=${BOSKOS_HOST:-"boskos.test-pods.svc.cluster.local."}
 
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
-# If BOSKOS_HOST is set then acquire an AWS account from Boskos.
-if [ -n "${BOSKOS_HOST:-}" ]; then
-  # Check out the account from Boskos and store the produced environment
-  # variables in a temporary file.
-  account_env_var_file="$(mktemp)"
-  python hack/checkout_account.py 1>"${account_env_var_file}"
-  checkout_account_status="${?}"
-
-  # If the checkout process was a success then load the account's
-  # environment variables into this process.
-  # shellcheck disable=SC1090
-  [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
-
-  # Always remove the account environment variable file. It contains
-  # sensitive information.
-  rm -f "${account_env_var_file}"
-
-  if [ ! "${checkout_account_status}" = "0" ]; then
-    echo "error getting account from boskos" 1>&2
-    exit "${checkout_account_status}"
-  fi
+echo "using boskos host to checkout project: ${BOSKOS_HOST}"
+
+# If BOSKOS_HOST is set then acquire an GCP account from Boskos.
+# Check out the account from Boskos and store the produced environment
+# variables in a temporary file.
+account_env_var_file="$(mktemp)"
+python hack/checkout_account.py 1>"${account_env_var_file}"
+checkout_account_status="${?}"
+
+# If the checkout process was a success then load the account's
+# environment variables into this process.
+# shellcheck disable=SC1090
+[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+
+# Always remove the account environment variable file. It contains
+# sensitive information.
+rm -f "${account_env_var_file}"
+
+if [ ! "${checkout_account_status}" = "0" ]; then
+  echo "error getting account from boskos" 1>&2
+  exit "${checkout_account_status}"
 fi
 
 (cd "${REPO_ROOT}" && hack/ci/e2e-conformance.sh)
+
+echo "cleaning up - checking in boskos account $BOSKOS_RESOURCE_NAME on host $BOSKOS_HOST"
+hack/checkin_account.py