Refactor release process for Github and Staging

Replace use of sed with gojq to modify templates, by copying
manifests to _artifacts and outputting to out/*-components.yaml,
preventing source files being modified in place.

Switch to using Docker build cache to speed up builds, should
reduce CloudBuild times significantly.

CloudBuild modified such that manifests will be generated and uploaded
to GCS on execution. This will allow advanced users to be able to test
manifests directly from staging for testing prior to release.

Finally, targets are added to create a release in Github using the
Github CLI.

Signed-off-by: Naadir Jeewa <jeewan@vmware.com>

Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:experimental
+
 # Copyright 2019 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,16 +33,15 @@ RUN go mod download
 COPY ./ ./
 
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \
-    wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \
-    chmod +x /start.sh && chmod +x /restart.sh
+  wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \
+  chmod +x /start.sh && chmod +x /restart.sh
 
 # Build
 ARG package=.
 ARG ARCH
 ARG LDFLAGS
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
-    go build -a -ldflags "${LDFLAGS} -extldflags '-static'" \
-    -o manager ${package}
+RUN --mount=type=cache,target=/root/.cache/go-build \
+  CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags "${LDFLAGS} -extldflags '-static'"  -o manager ${package}
 ENTRYPOINT [ "/start.sh", "/workspace/manager" ]
 
 # Copy the controller-manager into a thin image

common.mk

@@ -17,9 +17,9 @@ include $(ROOT_DIR_RELATIVE)/versions.mk
 # Ensure Make is run with bash shell as some syntax below is bash-specific
 SHELL:=bash
 .ONESHELL:
+.EXPORT_ALL_VARIABLES:
 .SHELLFLAGS := -eu -o pipefail -c
 .DELETE_ON_ERROR:
-MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 
 TOOLS_DIR := $(ROOT_DIR_RELATIVE)/hack/tools

docs/book/src/development/releasing.md

@@ -4,25 +4,24 @@
 
 1. Make sure your repo is clean by git's standards
 2. If this is a new minor release, create a new release branch and push to github, otherwise switch to it, for example `release-0.2`
-3. Run `make release-notes` to gather changes since the last revision. If you need to specify a specific tag to look for changes
-   since, use `make release-notes ARGS="--from <tag>"` Pay close attention to the `## :question: Sort these by hand` section, as it contains items that need to be manually sorted.
-4. Tag the repository and push the tag `git tag -s -m $VERSION $VERSION`
-5. Create a draft release in github and associate it with the tag that was just created, copying the generated release notes into
-   the draft.
-6. Checkout the tag you've just created and make sure git is in a clean state
-7. Run `make release`
-8. Attach the files to the drafted release:
+3. Tag the repository and push the tag `git tag -s -m $VERSION $VERSION`
+4. Set an environment variable PREVIOUS_VERSION to the last release tag.
+5. Checkout the tag you've just created and make sure git is in a clean state
+6. Run `make release`
+7. Run `make create-gh-release` to create a draft release on Github, copying the generated release notes from out/CHANGELOG.md into the draft.
+8. Run `make upload-gh-artifacts` to upload artifacts, however you may run into API limit errors, so verify artifacts at next step
+9. Attach the files to the drafted release:
     1. `./out/clusterawsadm-darwin-amd64`
     2. `./out/clusterawsadm-linux-amd64`
     3. `./out/infrastructure-components.yaml`
     4. `./out/cluster-template.yaml`
     5. `./out/cluster-template-machinepool.yaml`
     6. `./out/cluster-template-eks.yaml`
     7. `./out/cluster-template-eks-managedmachinepool.yaml`
-9.  Perform the [image promotion process](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io#image-promoter).
+10. Perform the [image promotion process](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io#image-promoter).
     The staging repository is at https://console.cloud.google.com/gcr/images/k8s-staging-cluster-api-aws/GLOBAL. Be
     sure to choose the top level `cluster-api-aws-controller`, which will provide the multi-arch manifest, rather than one for a specific architecture.
-10.  Finalise the release notes
-11.  Publish release. Use the pre-release option for release
+11.  Finalise the release notes
+12.  Publish release. Use the pre-release option for release
     candidate versions of Cluster API Provider AWS.
-12.  Email `kubernetes-sig-cluster-lifecycle@googlegroups.com` to announce the release
+13.  Email `kubernetes-sig-cluster-lifecycle@googlegroups.com` to announce the release

hack/boilerplate/boilerplate.Dockerfile.txt

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:experimental
+
 # Copyright YEAR The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

hack/image-patch/kustomization.yaml

@@ -0,0 +1,23 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+images:
+  - name: ""
+    newName: ""
+    newTag: ""
+kind: Kustomization
+patchesJson6902:
+  - path: pull-policy-patch.yaml
+    target:
+      group: apps
+      kind: Deployment
+      name: controller-name
+      namespace: namespace
+      version: v1
+  - path: pull-policy-patch.yaml
+    target:
+      group: apps
+      kind: Deployment
+      name: controller-name
+      namespace: capi-webhook-system
+      version: v1
+resources:
+  - source-manifest.yaml

hack/image-patch/pull-policy-patch.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/template/spec/containers/0/imagePullPolicy
+  value: Always

hack/tools/Makefile

@@ -23,6 +23,18 @@ SHARE_DIR := share
 
 OS := $(shell go env GOOS)
 RUST_TARGET := unknown-$(OS)-gnu
+
+ifeq ($(OS), darwin)
+	RUST_TARGET := apple-darwin
+	GH_ARCH_SUFFIX := macOS_amd64
+	GTAR := gtar
+endif
+
+ifeq ($(OS), linux)
+	GH_ARCH_SUFFIX := linux_amd64
+	GTAR := tar
+endif
+
 MDBOOK_EXTRACT_COMMAND := tar xfvz $(SHARE_DIR)/mdbook.tar.gz -C bin
 MDBOOK_ARCHIVE_EXT := .tar.gz
 
@@ -32,10 +44,6 @@ ifeq ($(OS), windows)
 	MDBOOK_EXTRACT_COMMAND := unzip -d /tmp
 endif
 
-ifeq ($(OS), darwin)
-	RUST_TARGET := apple-darwin
-endif
-
 ## --------------------------------------
 ## Tooling Binaries
 ## --------------------------------------
@@ -46,6 +54,10 @@ $(BIN_DIR):
 $(SHARE_DIR):
 	mkdir -p $@
 
+.PHONY: $(GTAR)
+$(GTAR):
+	@$(GTAR) --version > /dev/null || (echo Install GNU Tar with brew install gnu-tar && exit -1)
+
 CONTROLLER_GEN := $(BIN_DIR)/controller-gen
 $(CONTROLLER_GEN): $(BIN_DIR) go.mod go.sum # Build controller-gen from tools folder.
 	go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen
@@ -62,10 +74,28 @@ ENVSUBST := $(BIN_DIR)/envsubst
 $(ENVSUBST): $(BIN_DIR) go.mod go.sum # Build envsubst from tools folder.
 	go build -tags=tools -o $@ github.com/a8m/envsubst/cmd/envsubst
 
+GH_SHARE := $(SHARE_DIR)/gh
+
+$(GH_SHARE): $(SHARE_DIR)
+	mkdir -p $@
+
+$(GH_SHARE)/gh.tar.gz: $(GH_SHARE)
+	curl -L "https://github.com/cli/cli/releases/download/v$(GH_VERSION)/gh_$(GH_VERSION)_$(GH_ARCH_SUFFIX).tar.gz" -o $@
+
+GH := $(BIN_DIR)/gh
+$(GH): $(GH_SHARE)/gh.tar.gz
+	$(GTAR) -xvf share/gh/gh.tar.gz gh_$(GH_VERSION)_$(GH_ARCH_SUFFIX)/bin/gh --strip-components 1 --directory $(BIN_DIR)
+	chmod +x $@
+	touch -m $@
+
 GINKGO := $(BIN_DIR)/ginkgo
 $(GINKGO): $(BIN_DIR) go.mod go.sum
 	go build -tags=tools -o $@ github.com/onsi/ginkgo/ginkgo
 
+GOJQ := $(BIN_DIR)/gojq
+$(GOJQ): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ github.com/itchyny/gojq/cmd/gojq
+
 GOLANGCI_LINT := $(BIN_DIR)/golangci-lint
 $(GOLANGCI_LINT): $(BIN_DIR) go.mod go.sum # Build golangci-lint from tools folder.
 	go build -tags=tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint
@@ -135,7 +165,7 @@ SSM_PLUGIN := $(BIN_DIR)/session-manager-plugin
 $(SSM_PLUGIN): $(BIN_DIR)
 ifeq ($(UNAME), Linux)
 	$(MAKE) $(SSM_SHARE)/data.tar.gz
-	cd $(BIN_DIR) && tar -xvf ../share/ssm/data.tar.gz usr/local/sessionmanagerplugin/bin/session-manager-plugin --strip-components 4 --directory $(BIN_DIR)
+	cd $(BIN_DIR) && $(GTAR) -xvf ../share/ssm/data.tar.gz usr/local/sessionmanagerplugin/bin/session-manager-plugin --strip-components 4 --directory $(BIN_DIR)
 endif
 ifeq ($(UNAME), Darwin)
 	$(MAKE) $(SSM_SHARE)/sessionmanager-bundle.zip

hack/tools/go.mod

@@ -6,11 +6,15 @@ require (
 	github.com/a8m/envsubst v1.2.0
 	github.com/golang/mock v1.4.4
 	github.com/golangci/golangci-lint v1.31.0
+	github.com/itchyny/gojq v0.11.2
 	github.com/onsi/ginkgo v1.14.1
+	github.com/spf13/cobra v1.1.1 // indirect
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
+	golang.org/x/text v0.3.3 // indirect
 	k8s.io/code-generator v0.18.8
 	sigs.k8s.io/cluster-api/hack/tools v0.0.0-20201001010053-9e2a7c7a886a
 	sigs.k8s.io/controller-tools v0.2.9
 	sigs.k8s.io/kind v0.6.1
-	sigs.k8s.io/kustomize/kustomize/v3 v3.5.4
+	sigs.k8s.io/kustomize/kustomize/v3 v3.8.6
 	sigs.k8s.io/testing_frameworks v0.1.2
 )