[action-runners] Add k8s action runner with KVM support

k8s/devinfra/action-runners/README.md

@@ -18,7 +18,17 @@ helm upgrade --install --namespace actions-runner-system --create-namespace\
   --set=authSecret.github_token="REPLACE_YOUR_TOKEN_HERE"\
   --wait actions-runner-controller actions-runner-controller/actions-runner-controller
 ```
+### Create the runners namespace
+```
+kubectl create namespace action-runners
+```
 
-### Deploy the Runner
+### Create the buildbuddy secret
+```
+BB_API_KEY=<BUILDBUDDY-API-KEY> envsubst < k8s/devinfra/action-runners/bb_bazelrc_secret.yaml | kubectl apply -f -
+```
 
-`kubectl apply -f runnerdeployment.yaml`
+### Deploy the Runners
+```
+kustomize build k8s/devinfra/action-runners/runners | kubectl apply -f -
+```

k8s/devinfra/action-runners/bb_bazelrc_secret.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: bb-bazelrc
+  namespace: action-runners
+type: Opaque
+stringData:
+  bazelrc: |
+    build --remote_header=x-buildbuddy-api-key=${BB_API_KEY}

k8s/devinfra/action-runners/runners/base/autoscaler.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: runner-autoscaler
+spec:
+  scaleTargetRef:
+    kind: RunnerDeployment
+    name: runner
+  minReplicas: 1
+  maxReplicas: 32
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - pixie

k8s/devinfra/action-runners/runners/base/deployment.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: runner
+spec:
+  template:
+    spec:
+      repository: pixie-io/pixie
+      resources:
+        requests:
+          cpu: 16000m
+      volumeMounts:
+      - mountPath: /etc/bazelrc
+        subPath: bazelrc
+        name: bb-bazelrc
+      dockerVolumeMounts:
+      - mountPath: /etc/docker/daemon.json
+        subPath: daemon.json
+        name: dockerd-config
+      volumes:
+      - name: bb-bazelrc
+        secret:
+          secretName: bb-bazelrc
+      - name: dockerd-config
+        configMap:
+          name: dockerd-config

k8s/devinfra/action-runners/runners/base/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- autoscaler.yaml
+- deployment.yaml
+configurations:
+- name_reference.yaml

k8s/devinfra/action-runners/runners/base/name_reference.yaml

@@ -0,0 +1,6 @@
+---
+nameReference:
+- kind: RunnerDeployment
+  fieldSpecs:
+  - kind: HorizontalRunnerAutoscaler
+    path: spec/scaleTargetRef/name

k8s/devinfra/action-runners/runners/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: action-runners
+resources:
+- shared
+- kvm
+- nokvm

k8s/devinfra/action-runners/runners/kvm/deployment.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: runner
+spec:
+  template:
+    spec:
+      labels:
+      - kvm
+      nodeSelector:
+        nested-virt: enabled
+      tolerations:
+      - key: "nested-virt"
+        operator: "Equal"
+        value: "enabled"
+        effect: "NoSchedule"

k8s/devinfra/action-runners/runners/kvm/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../base
+nameSuffix: -kvm
+patches:
+- path: deployment.yaml
+  target:
+    kind: RunnerDeployment
+    name: runner

k8s/devinfra/action-runners/runnerdeployment.yaml → k8s/devinfra/action-runners/runners/nokvm/deployment.yaml

@@ -2,10 +2,9 @@
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
-  name: pixie-runnerdeploy
-  namespace: actions-runner-system
+  name: runner
 spec:
-  replicas: 1
   template:
     spec:
-      repository: pixie-io/pixie
+      labels:
+      - nokvm

k8s/devinfra/action-runners/runners/nokvm/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../base
+nameSuffix: -nokvm
+patches:
+- path: deployment.yaml
+  target:
+    kind: RunnerDeployment
+    name: runner

k8s/devinfra/action-runners/runners/shared/docker_config.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dockerd-config
+data:
+  # yamllint disable rule:indentation
+  daemon.json: |
+    {
+      "ipv6": true,
+      "fixed-cidr-v6": "2001:db8:1::/64"
+    }
+  # yamllint enable rule:indentation

k8s/devinfra/action-runners/runners/shared/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- docker_config.yaml