diff --git a/k8s/devinfra/action-runners/README.md b/k8s/devinfra/action-runners/README.md index 6fb029516ff..39782e952b7 100644 --- a/k8s/devinfra/action-runners/README.md +++ b/k8s/devinfra/action-runners/README.md @@ -18,7 +18,17 @@ helm upgrade --install --namespace actions-runner-system --create-namespace\ --set=authSecret.github_token="REPLACE_YOUR_TOKEN_HERE"\ --wait actions-runner-controller actions-runner-controller/actions-runner-controller ``` +### Create the runners namespace +``` +kubectl create namespace action-runners +``` -### Deploy the Runner +### Create the buildbuddy secret +``` +BB_API_KEY= envsubst < k8s/devinfra/action-runners/bb_bazelrc_secret.yaml | kubectl apply -f - +``` -`kubectl apply -f runnerdeployment.yaml` +### Deploy the Runners +``` +kustomize build k8s/devinfra/action-runners/runners | kubectl apply -f - +``` diff --git a/k8s/devinfra/action-runners/bb_bazelrc_secret.yaml b/k8s/devinfra/action-runners/bb_bazelrc_secret.yaml new file mode 100644 index 00000000000..60a793e1b43 --- /dev/null +++ b/k8s/devinfra/action-runners/bb_bazelrc_secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: bb-bazelrc + namespace: action-runners +type: Opaque +stringData: + bazelrc: | + build --remote_header=x-buildbuddy-api-key=${BB_API_KEY} diff --git a/k8s/devinfra/action-runners/runners/base/autoscaler.yaml b/k8s/devinfra/action-runners/runners/base/autoscaler.yaml new file mode 100644 index 00000000000..998a754cf0b --- /dev/null +++ b/k8s/devinfra/action-runners/runners/base/autoscaler.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: runner-autoscaler +spec: + scaleTargetRef: + kind: RunnerDeployment + name: runner + minReplicas: 1 + maxReplicas: 32 + metrics: + - type: TotalNumberOfQueuedAndInProgressWorkflowRuns + repositoryNames: + - pixie diff --git a/k8s/devinfra/action-runners/runners/base/deployment.yaml b/k8s/devinfra/action-runners/runners/base/deployment.yaml new file mode 100644 index 00000000000..f349f9f50de --- /dev/null +++ b/k8s/devinfra/action-runners/runners/base/deployment.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runner +spec: + template: + spec: + repository: pixie-io/pixie + resources: + requests: + cpu: 16000m + volumeMounts: + - mountPath: /etc/bazelrc + subPath: bazelrc + name: bb-bazelrc + dockerVolumeMounts: + - mountPath: /etc/docker/daemon.json + subPath: daemon.json + name: dockerd-config + volumes: + - name: bb-bazelrc + secret: + secretName: bb-bazelrc + - name: dockerd-config + configMap: + name: dockerd-config diff --git a/k8s/devinfra/action-runners/runners/base/kustomization.yaml b/k8s/devinfra/action-runners/runners/base/kustomization.yaml new file mode 100644 index 00000000000..b272ca86bd7 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/base/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- autoscaler.yaml +- deployment.yaml +configurations: +- name_reference.yaml diff --git a/k8s/devinfra/action-runners/runners/base/name_reference.yaml b/k8s/devinfra/action-runners/runners/base/name_reference.yaml new file mode 100644 index 00000000000..7f2567633c0 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/base/name_reference.yaml @@ -0,0 +1,6 @@ +--- +nameReference: +- kind: RunnerDeployment + fieldSpecs: + - kind: HorizontalRunnerAutoscaler + path: spec/scaleTargetRef/name diff --git a/k8s/devinfra/action-runners/runners/kustomization.yaml b/k8s/devinfra/action-runners/runners/kustomization.yaml new file mode 100644 index 00000000000..3603bc76075 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: action-runners +resources: +- shared +- kvm +- nokvm diff --git a/k8s/devinfra/action-runners/runners/kvm/deployment.yaml b/k8s/devinfra/action-runners/runners/kvm/deployment.yaml new file mode 100644 index 00000000000..a1dc3e95af5 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/kvm/deployment.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runner +spec: + template: + spec: + labels: + - kvm + nodeSelector: + nested-virt: enabled + tolerations: + - key: "nested-virt" + operator: "Equal" + value: "enabled" + effect: "NoSchedule" diff --git a/k8s/devinfra/action-runners/runners/kvm/kustomization.yaml b/k8s/devinfra/action-runners/runners/kvm/kustomization.yaml new file mode 100644 index 00000000000..305031583bd --- /dev/null +++ b/k8s/devinfra/action-runners/runners/kvm/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../base +nameSuffix: -kvm +patches: +- path: deployment.yaml + target: + kind: RunnerDeployment + name: runner diff --git a/k8s/devinfra/action-runners/runnerdeployment.yaml b/k8s/devinfra/action-runners/runners/nokvm/deployment.yaml similarity index 50% rename from k8s/devinfra/action-runners/runnerdeployment.yaml rename to k8s/devinfra/action-runners/runners/nokvm/deployment.yaml index 627e96b3a79..04fa9eb0590 100644 --- a/k8s/devinfra/action-runners/runnerdeployment.yaml +++ b/k8s/devinfra/action-runners/runners/nokvm/deployment.yaml @@ -2,10 +2,9 @@ apiVersion: actions.summerwind.dev/v1alpha1 kind: RunnerDeployment metadata: - name: pixie-runnerdeploy - namespace: actions-runner-system + name: runner spec: - replicas: 1 template: spec: - repository: pixie-io/pixie + labels: + - nokvm diff --git a/k8s/devinfra/action-runners/runners/nokvm/kustomization.yaml b/k8s/devinfra/action-runners/runners/nokvm/kustomization.yaml new file mode 100644 index 00000000000..7356e9dc3ef --- /dev/null +++ b/k8s/devinfra/action-runners/runners/nokvm/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../base +nameSuffix: -nokvm +patches: +- path: deployment.yaml + target: + kind: RunnerDeployment + name: runner diff --git a/k8s/devinfra/action-runners/runners/shared/docker_config.yaml b/k8s/devinfra/action-runners/runners/shared/docker_config.yaml new file mode 100644 index 00000000000..03dbe79a0f8 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/shared/docker_config.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dockerd-config +data: + # yamllint disable rule:indentation + daemon.json: | + { + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + } + # yamllint enable rule:indentation diff --git a/k8s/devinfra/action-runners/runners/shared/kustomization.yaml b/k8s/devinfra/action-runners/runners/shared/kustomization.yaml new file mode 100644 index 00000000000..9e0c5f23865 --- /dev/null +++ b/k8s/devinfra/action-runners/runners/shared/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- docker_config.yaml