feat: Support for scaling from/to zero

Makefile

@@ -113,7 +113,7 @@ generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
 
 # Build the docker image
-docker-build: test
+docker-build:
 	docker build . -t ${NAME}:${VERSION}
 	docker build runner -t ${RUNNER_NAME}:${VERSION} --build-arg TARGETPLATFORM=$(shell arch)
 
@@ -172,7 +172,7 @@ acceptance/pull: docker-build
 
 acceptance/setup:
 	kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.yaml	#kubectl create namespace actions-runner-system
-	kubectl -n cert-manager wait deploy/cert-manager-cainjector --for condition=available --timeout 60s
+	kubectl -n cert-manager wait deploy/cert-manager-cainjector --for condition=available --timeout 90s
 	kubectl -n cert-manager wait deploy/cert-manager-webhook --for condition=available --timeout 60s
 	kubectl -n cert-manager wait deploy/cert-manager --for condition=available --timeout 60s
 	kubectl create namespace actions-runner-system || true
@@ -230,6 +230,7 @@ OS_NAME := $(shell uname -s | tr A-Z a-z)
 
 # find or download etcd
 etcd:
+ifeq (, $(shell which etcd))
 ifeq (, $(wildcard $(TEST_ASSETS)/etcd))
 	@{ \
 	set -xe ;\
@@ -247,9 +248,13 @@ ETCD_BIN=$(TEST_ASSETS)/etcd
 else
 ETCD_BIN=$(TEST_ASSETS)/etcd
 endif
+else
+ETCD_BIN=$(shell which etcd)
+endif
 
 # find or download kube-apiserver
 kube-apiserver:
+ifeq (, $(shell which kube-apiserver))
 ifeq (, $(wildcard $(TEST_ASSETS)/kube-apiserver))
 	@{ \
 	set -xe ;\
@@ -267,10 +272,13 @@ KUBE_APISERVER_BIN=$(TEST_ASSETS)/kube-apiserver
 else
 KUBE_APISERVER_BIN=$(TEST_ASSETS)/kube-apiserver
 endif
-
+else
+KUBE_APISERVER_BIN=$(shell which kube-apiserver)
+endif
 
 # find or download kubectl
 kubectl:
+ifeq (, $(shell which kubectl))
 ifeq (, $(wildcard $(TEST_ASSETS)/kubectl))
 	@{ \
 	set -xe ;\
@@ -288,3 +296,6 @@ KUBECTL_BIN=$(TEST_ASSETS)/kubectl
 else
 KUBECTL_BIN=$(TEST_ASSETS)/kubectl
 endif
+else
+KUBECTL_BIN=$(shell which kubectl)
+endif

README.md

@@ -17,6 +17,7 @@ ToC:
   - [Organization Runners](#organization-runners)
   - [Enterprise Runners](#enterprise-runners)
   - [Runner Deployments](#runnerdeployments)
+    - [Note on scaling to/from 0](#note-on-scaling-to-from-zero)
     - [Autoscaling](#autoscaling)
       - [Faster Autoscaling with GitHub Webhook](#faster-autoscaling-with-github-webhook)
   - [Runner with DinD](#runner-with-dind)
@@ -279,6 +280,22 @@ example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 ```
 
+##### Note on scaling to/from 0
+
+You can either delete the runner deployment, or update it to have `replicas: 0`, so that there will be 0 runner pods in the cluster. This, in combination with e.g. `cluster-autoscaler`, enables you to save your infrastructure cost when there's no need to run Actions jobs.
+
+```yaml
+# runnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  replicas: 0
+```
+
+The implication of setting `replicas: 0` instead of deleting the runner depoyment is that you can let GitHub Actions queue jobs until there will be one or more runners. See [#465](https://github.com/actions-runner-controller/actions-runner-controller/pull/465) for more information.
+
 #### Autoscaling
 
 __**IMPORTANT : Due to limitations / a bug with GitHub's [routing engine](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#routing-precedence-for-self-hosted-runners) autoscaling does NOT work correctly with RunnerDeployments that target the enterprise level. Scaling activity works as expected however jobs fail to get assigned to the scaled out replicas. This was explored in issue [#470](https://github.com/actions-runner-controller/actions-runner-controller/issues/470). Once GitHub resolves the issue with their backend service we expect the solution to be able to support autoscaled enterprise runnerdeploments without any additional changes.**__
@@ -816,5 +833,50 @@ KUBECONFIG=path/to/kubeconfig \
        acceptance/tests
 ```
 
+**Development Tips**
+
+If you've already deployed actions-runner-controller and only want to recreate pods to use the newer image, you can run:
+
+```
+NAME=$DOCKER_USER/actions-runner-controller \
+  make docker-build docker-push && \
+  kubectl -n actions-runner-system delete po $(kubectl -n actions-runner-system get po -ojsonpath={.items[*].metadata.name})
+```
+
+Similary, if you'd like to recreate runner pods with the newer runner image,
+
+```
+NAME=$DOCKER_USER/actions-runner make \
+  -C runner docker-{build,push}-ubuntu && \
+  (kubectl get po -ojsonpath={.items[*].metadata.name} | xargs -n1 kubectl delete po)
+```
+
 **Runner Tests**<br />
 A set of example pipelines (./acceptance/pipelines) are provided in this repository which you can use to validate your runners are working as expected. When raising a PR please run the relevant suites to prove your change hasn't broken anything.
+
+**Running Ginkgo Tests**
+
+You can run the integration test suite that is written in Ginkgo with:
+
+```bash
+make test-with-deps
+```
+
+This will firstly install a few binaries required to setup the integration test environment and then runs `go test` to start the Ginkgo test.
+
+If you don't want to use `make`, like when you're running tests from your IDE, install required binaries to `/usr/local/kubebuilder/bin`. That's the directory in which controller-runtime's `envtest` framework locates the binaries.
+
+```bash
+sudo mkdir -p /usr/local/kubebuilder/bin
+make kube-apiserver etcd
+sudo mv test-assets/{etcd,kube-apiserver} /usr/local/kubebuilder/bin/
+go test -v -run TestAPIs github.com/summerwind/actions-runner-controller/controllers
+```
+
+To run Ginkgo tests selectively, set the pattern of target test names to `GINKGO_FOCUS`.
+All the Ginkgo test that matches `GINKGO_FOCUS` will be run.
+
+```bash
+GINKGO_FOCUS='[It] should create a new Runner resource from the specified template, add a another Runner on replicas increased, and removes all the replicas when set to 0' \
+  go test -v -run TestAPIs github.com/summerwind/actions-runner-controller/controllers
+```

acceptance/testdata/runnerdeploy.yaml

@@ -7,6 +7,10 @@ spec:
   template:
     spec:
       repository: ${TEST_REPO}
+
+      #
+      # Custom runner image
+      #
       image: ${RUNNER_NAME}:${VERSION}
       imagePullPolicy: IfNotPresent
 
@@ -21,3 +25,13 @@ spec:
       # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
       #
       #dockerMTU: 1450
+
+      #Runner group
+      # labels:
+      # - "mylabel 1"
+      # - "mylabel 2"
+
+      #
+      # Non-standard working directory
+      #
+      # workDir: "/"

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: summerwind/actions-runner-controller
-  newTag: latest
+  newName: mumoshu/actions-runner-controller
+  newTag: dev

controllers/integration_test.go

@@ -446,9 +446,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				ExpectCreate(ctx, rd, "test RunnerDeployment")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
 			}
 
@@ -554,9 +551,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				ExpectCreate(ctx, rd, "test RunnerDeployment")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
 			}
 
@@ -595,9 +589,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
 			}
 
@@ -606,19 +597,15 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				env.SendOrgCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
 			}
 
 			// Scale-up to 3 replicas on second check_run create webhook event
 			{
 				env.SendOrgCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
 			}
-
-			env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
 		})
 
 		It("should create and scale user's repository runners on pull_request event", func() {
@@ -884,9 +871,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				ExpectCreate(ctx, rd, "test RunnerDeployment")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
 			}
 
@@ -930,9 +914,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3)
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
 			}
 
@@ -941,19 +922,15 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				env.SendUserCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 4, "runners after first webhook event")
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(4, "count of fake list runners")
 			}
 
 			// Scale-up to 5 replicas on second check_run create webhook event
 			{
 				env.SendUserCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 5, "runners after second webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(5, "count of fake list runners")
 			}
-
-			env.ExpectRegisteredNumberCountEventuallyEquals(5, "count of fake list runners")
 		})
 
 		It("should create and scale user's repository runners only on check_run event", func() {
@@ -1045,19 +1022,15 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				env.SendUserCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
-			}
-
-			{
 				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
 			}
 
 			// Scale-up to 3 replicas on second check_run create webhook event
 			{
 				env.SendUserCheckRunEvent("test", "valid", "pending", "created")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
 			}
-
-			env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
 		})
 
 	})