eks-prow-build-cluster: GitOps proposal

infra/aws/terraform/prow-build-cluster/Makefile

@@ -15,46 +15,94 @@
 TF ?= terraform
 TF_ARGS ?=
 
+KUBECTL ?= kubectl
+
 # TODO: figure out a workaround for: https://github.com/hashicorp/terraform-provider-kubernetes-alpha/issues/199#issuecomment-832614387
 DEPLOY_K8S_RESOURCES ?= true
 
 # Valid values are: canary, prod
 PROW_ENV ?= canary
 
+##@ Helpers:
+
+.PHONY: help
+help: ## Display this help.
+	@awk \
+		-v "col=${COLOR}" -v "nocol=${NOCOLOR}" \
+		' \
+			BEGIN { \
+				FS = ":.*##" ; \
+				printf "\nUsage:\n  make %s<target>%s\n", col, nocol \
+			} \
+			/^[a-zA-Z_-]+:.*?##/ { \
+				printf "  %s%-15s%s %s\n", col, $$1, nocol, $$2 \
+			} \
+			/^##@/ { \
+				printf "\n%s%s%s\n", col, substr($$0, 5), nocol \
+			} \
+		' $(MAKEFILE_LIST)
+
+##@ Terraform:
+
 .PHONY: init
-init:
+init: ## Initialize Terraform's state and download necessary providers.
 	$(TF) $@ \
 		-backend-config=./tfbackends/$(PROW_ENV).tfbackend
 
 .PHONY: plan
-plan:
+plan: ## Present plan for creating/updating Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: apply
-apply:
+apply: ## Create/Update Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: destroy
-destroy:
+destroy: ## Delete Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: fmt
-fmt:
+fmt: ## Format Terraform files.
 	$(TF) $@
 
 .PHONY: output
-output:
+output: ## Print Terraform output.
 	$(TF) $@
 
 .PHONY: clean
-clean:
+clean: ## Clean up Terraform cache and local state.
 	rm -rf ./.terraform
+
+##@ FluxCD:
+
+.PHONY: flux-install
+flux-install: ## Install FluxCD components inside flux-system namespace.
+	kubectl apply --server-side -f ./resources/flux-system/gotk-components.yaml
+
+.PHONY: flux-apply-sources
+flux-apply-sources: ## Apply FluxCD Sources located inside "./resources/flux-system" folder.
+	find ./resources/flux-system -type f -name "flux-source-*" -exec ${KUBECTL} apply -f {} \;
+
+.PHONY: flux-apply-kustomizations
+flux-apply-kustomizations: ## Apply FluxCD Kustomizations located inside "./resources/flux-system" folder.
+	find ./resources/flux-system -type f -name "flux-ks-*" -exec ${KUBECTL} apply -f {} \;
+
+.PHONY: flux-apply-helm-releases
+flux-apply-helm-releases: ## Apply HelmReleases located inside "./resources" folder.
+	find ./resources -type f -name "flux-hr-*" -exec ${KUBECTL} apply -f {} \;
+
+PHONY: flux-apply-all
+flux-apply-all: flux-apply-sources flux-apply-kustomizations flux-apply-helm-releases ## Apply FluxCD Sources, Kustomizations and HelmReleases, all at once.
+
+.PHONY: flux-update
+flux-update: ## Update FluxCD manifests, generate Kustomizations and HelmReleases (requires flux cli).
+	./hack/flux-update.bash

infra/aws/terraform/prow-build-cluster/README.md

@@ -243,3 +243,53 @@ make destroy
 
 If you want to remove roles used for EKS creation go to `../iam/<aws_account_name>` and run `terraform destroy` command there.
 
+
+## GitOps
+
+To synchronize the state from this Git repository into the EKS cluster, we utilize [FluxCD](https://fluxcd.io/).
+
+FluxCD provides a dedicated [CLI tool](https://fluxcd.io/flux/installation/#install-the-flux-cli). We leverage this tool in scripts to generate and monitor syncs.
+
+The FluxCD resources are stored inside the `./resources` directory and are generated using the `./hack/flux-update.bash` script. This script prepares manifests for the [GitOps Tool Kit](https://fluxcd.io/flux/components/), generating [Sources](https://fluxcd.io/flux/components/source/), [Kustomizations](https://fluxcd.io/flux/components/kustomize/kustomization/) and [Helm Releases](https://fluxcd.io/flux/components/helm/helmreleases/). **It's important to note that the terms "Kustomization" and "Helm Release" used in this section refer specifically to FluxCD concepts.** The FluxCD Kustomization CRD serves as the counterpart to Kustomize's kustomization.yaml config file, and the Helm Release is a FluxCD CRD that defines a resource for automated controller-driven Helm releases.
+
+The `flux-system` namespace contains all GitOps Tool Kit components, Flux Sources, and Kustomizations. Helm Releases should be deployed in the same namespace as the manifests they create. As a convention, Flux Helm Releases should be prefixed with `flux-hr-`, Kustomizations with `flux-ks-`, and sources with `flux-source-`. These naming conventions are used in our automation's discovery process.
+
+- To install Flux GitOps Tool Kit components, run the following command:
+    ```bash
+    make flux-install
+    ```
+
+- To deploy Kustomizations, use the command:
+    ```bash
+    make flux-apply-kustomizations
+    ```
+
+### Accessing Flux
+
+To access FluxCD, you need a kubeconfig file with broad cluster permissions and the Flux CLI installed locally. That option is reserved for cluster administrators. You can contanct them via #sig-k8s-infra Slack channel. (__TODO__: mention monitoring dashboards when ready)
+
+### Adding New Kustomization
+
+1. Navigate to the `./resources` directory and create a new folder to contain your manifests.
+2. Open `./hack/flux-update.bash` in your editor of choice.
+3. Locate the `kustomizations` variable, which contains a list of kustomizations to generate. Each element in the list corresonds to a directory inside the `./resources` folder.
+4. Extend the kustomization list by adding the name of the folder you created in the first step.
+5. Regenerate the kustomizations by running: `make flux-update`
+6. Commit your changes and wait for your resources to appear in the cluster. If you have access to the cluster, you can use `flux get ks <kustomization-name>` to monitor the state.
+
+### Adding New Helm Release
+
+1. Open `./hack/flux-update.bash` in your preferred editor.
+2. Create a new Helm source. If you already have a Helm source, you can skip this step.
+    1. Locate the section of the script responsible for creating the `eks-charts` Helm release.
+    2. Use the same pattern to create a new Helm source.
+3. Create a new Helm release.
+    1. Locate the section of the script responsible for creating the Helm release for `node-termination-handler`.
+    2. Based on the `node-termination-handler` example, extend the script with your Helm release.
+4. Regenerate the Helm releases by running: `make flux-update`
+5. To deploy your Helm Release, you need to add it to an exiting Kustomization or create a new one.
+6. If you have access to the cluster, you can check the status by executing: `flux get hr -n <namespace> <helm_release_name>` or `flux get hr -A` to list all Helm releases inside the cluster. (__TODO__: replace this section with grafana dashboard when ready)
+
+### Monitoring Flux
+
+__TODO__

infra/aws/terraform/prow-build-cluster/hack/flux-update.bash

@@ -0,0 +1,96 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+# TODO(pkprzekwas): point at k8s.io main
+github_org=pkprzekwas
+github_repo=k8s.io
+github_branch=eks-prow-build-cluster-gitops
+
+if ! command -v flux &> /dev/null
+then
+  echo "flux could not be found"
+  exit 2
+fi
+
+script_root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
+
+function boilerplate() {
+  cat ${script_root}/../../../../../hack/boilerplate/boilerplate.sh.txt | sed -e "s/\<YEAR\>/$(date +'%Y')/"
+  echo "# Code generated by make flux-update. DO NOT EDIT."
+}
+
+resources_dir=${script_root}/../resources
+
+# Generate all FluxCD resources.
+# gotk stands for GitOpsToolKit (https://fluxcd.io/flux/components/)
+boilerplate > ${resources_dir}/flux-system/gotk-components.yaml
+flux install --export >> ${resources_dir}/flux-system/gotk-components.yaml
+
+# sync_interval determines Flux Source Controller sync interval.
+# (https://github.com/fluxcd/source-controller)
+sync_interval=5m
+
+boilerplate > ${resources_dir}/flux-system/flux-source-git-k8s.io.yaml
+flux create source git k8s-io \
+    --url=https://github.com/${github_org}/k8s.io \
+    --branch=${github_branch} \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/flux-system/flux-source-git-k8s.io.yaml
+
+boilerplate > ${resources_dir}/flux-system/flux-source-helm-eks-charts.yaml
+flux create source helm eks-charts \
+    --url=https://aws.github.io/eks-charts \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/flux-system/flux-source-helm-eks-charts.yaml
+
+boilerplate > ${resources_dir}/kube-system/flux-hr-node-termination-handler.yaml
+flux create hr node-termination-handler \
+    --source=HelmRepository/eks-charts.flux-system \
+    --namespace=kube-system \
+    --chart=aws-node-termination-handler \
+    --chart-version=0.21.0 \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/kube-system/flux-hr-node-termination-handler.yaml
+
+# This list contains names of folders inside ./resources directory
+# that are used for generating FluxCD kustomizations.
+kustomizations=(
+    boskos
+    flux-system
+    kube-system
+    monitoring
+    node-problem-detector
+    rbac
+    test-pods
+    external-secrets
+)
+
+# Code below is used to figure out a relative path of
+# resources dir in relation to the root dir of git repo.
+pushd ${resources_dir} > /dev/null
+resources_git_repo_path=$(git rev-parse --show-prefix)
+popd > /dev/null
+
+for k in "${kustomizations[@]}"; do
+    boilerplate > ${resources_dir}/flux-system/flux-ks-${k}.yaml
+    flux create kustomization ${k} \
+        --source=GitRepository/k8s-io.flux-system \
+        --path=${resources_git_repo_path}/${k} \
+        --interval=5m \
+        --prune \
+        --export >> ${resources_dir}/flux-system/flux-ks-${k}.yaml
+done

infra/aws/terraform/prow-build-cluster/resources/external-secrets/namespace.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-secrets

infra/aws/terraform/prow-build-cluster/resources/flux-system/flux-ks-boskos.yaml

@@ -0,0 +1,29 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code generated by make flux-update. DO NOT EDIT.
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: boskos
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: ./infra/aws/terraform/prow-build-cluster/resources/boskos
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-io
+    namespace: flux-system

infra/aws/terraform/prow-build-cluster/resources/flux-system/flux-ks-external-secrets.yaml

@@ -0,0 +1,29 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code generated by make flux-update. DO NOT EDIT.
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: ./infra/aws/terraform/prow-build-cluster/resources/external-secrets
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-io
+    namespace: flux-system

infra/aws/terraform/prow-build-cluster/resources/flux-system/flux-ks-flux-system.yaml

@@ -0,0 +1,29 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code generated by make flux-update. DO NOT EDIT.
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: ./infra/aws/terraform/prow-build-cluster/resources/flux-system
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-io
+    namespace: flux-system

infra/aws/terraform/prow-build-cluster/resources/flux-system/flux-ks-kube-system.yaml

@@ -0,0 +1,29 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code generated by make flux-update. DO NOT EDIT.
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kube-system
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: ./infra/aws/terraform/prow-build-cluster/resources/kube-system
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-io
+    namespace: flux-system