Merge pull request #6 from dippynark/support-persistent-control-plane

Fix for newer kindest/node images and update documentation

Makefile

@@ -10,7 +10,7 @@ CONTROLLER_TOOLS_VERSION = v0.5.0
 
 CAPI_VERSION = v0.3.14
 CERT_MANAGER_VERSION = v0.16.1
-KUBERNETES_VERSION = v1.17.0
+KUBERNETES_VERSION = v1.20.2
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -29,20 +29,10 @@ manager: generate fmt vet
 run: generate fmt vet manifests
 	go run ./main.go
 
-# Install CRDs into a cluster
-install: manifests
-	kustomize build config/crd | kubectl apply -f -
-
-# Uninstall CRDs from a cluster
-uninstall: manifests
-	kustomize build config/crd | kubectl delete -f -
-
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: manifests
 	cd config/manager && kustomize edit set image controller=${IMG}
 	kustomize build config | kubectl apply -f -
-	# TODO: use aggregation label when available
-	kubectl apply -f release/kubeadm-control-plane-rbac.yaml
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
@@ -109,7 +99,7 @@ E2E_CONF_FILE  ?= $(CURDIR)/e2e/config/capk.yaml
 SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
 .PHONY: e2e
-e2e: $(GINKGO) docker-build e2e_template
+e2e: $(GINKGO) docker-build e2e_template e2e_data
 	cd config/manager && kustomize edit set image controller=${IMG}
 	$(GINKGO) -v -trace -tags=e2e ./e2e -- \
 		-e2e.artifacts-folder="$(ARTIFACTS)" \

README.md

@@ -1,11 +1,13 @@
 # Kubernetes Cluster API Provider Kubernetes
 
-The [Cluster API] brings declarative, Kubernetes-style APIs to cluster creation,
-configuration and management.
+The [Cluster API](https://github.com/kubernetes-sigs/cluster-api) brings declarative,
+Kubernetes-style APIs to cluster creation, configuration and management.
 
-This project is a [Cluster API Infrastructure Provider] implementation using
-Kubernetes itself to provide the infrastructure. Pods running [kind] are
-created and configured to serve as Nodes which form a cluster.
+This project is a [Cluster API Infrastructure
+Provider](https://cluster-api.sigs.k8s.io/reference/providers.html#infrastructure) implementation
+using Kubernetes itself to provide the infrastructure. Pods running
+[kind](https://github.com/kubernetes-sigs/kind) are created and configured to serve as Nodes which
+form a cluster.
 
 The primary use cases for this project are testing and experimentation.
 
@@ -20,24 +22,27 @@ Cluster API cluster as the inner cluster.
 
 Any recent Kubernetes cluster (1.16+) should be suitable for the outer cluster.
 
-We are going to use [Calico] as an overlay implementation for the inner cluster with [IP-in-IP
-encapsulation] enabled so that our outer cluster does not need to know about the inner cluster's Pod
-IP range. To make this work we need to ensure that the `ipip` kernel module is loadable and that
-IPv4 encapsulated packets are forwarded by the kernel.
+We are going to use [Calico](https://docs.projectcalico.org/v3.11/getting-started/kubernetes/) as an
+overlay implementation for the inner cluster with [IP-in-IP
+encapsulation](https://docs.projectcalico.org/v3.11/getting-started/kubernetes/installation/config-options#configuring-ip-in-ip)
+enabled so that our outer cluster does not need to know about the inner cluster's Pod IP range. To
+make this work we need to ensure that the `ipip` kernel module is loadable and that IPv4
+encapsulated packets are forwarded by the kernel.
 
 On GKE this can be accomplished as follows:
 
 ```sh
 # The GKE Ubuntu image includes the ipip kernel module
 # Calico handles loading the module if necessary
 # https://github.com/projectcalico/felix/blob/9469e77e0fa530523be915dfaa69cc42d30b8317/dataplane/linux/ipip_mgr.go#L107-L110
-gcloud container clusters create management \
+MANAGEMENT_CLUSTER_NAME="management"
+gcloud container clusters create $MANAGEMENT_CLUSTER_NAME \
   --image-type=UBUNTU \
   --machine-type=n1-standard-4
 
 # Allow IP-in-IP traffic between outer cluster Nodes from inner cluster Pods
-CLUSTER_CIDR=`gcloud container clusters describe management --format="value(clusterIpv4Cidr)"`
-gcloud compute firewall-rules create allow-management-cluster-pods-ipip \
+CLUSTER_CIDR=`gcloud container clusters describe $MANAGEMENT_CLUSTER_NAME --format="value(clusterIpv4Cidr)"`
+gcloud compute firewall-rules create allow-$MANAGEMENT_CLUSTER_NAME-cluster-pods-ipip \
   --source-ranges=$CLUSTER_CIDR \
   --allow=ipip
 
@@ -71,28 +76,30 @@ clusterctl init --infrastructure kubernetes
 ### Configuration
 
 ```sh
+CLUSTER_NAME="example"
+
 # Use ClusterIP for clusters that do not support Services of type LoadBalancer
 export KUBERNETES_CONTROL_PLANE_SERVICE_TYPE="LoadBalancer"
 export KUBERNETES_CONTROL_PLANE_MACHINE_CPU_REQUESTS="1"
 export KUBERNETES_CONTROL_PLANE_MACHINE_MEMORY_REQUESTS="1Gi"
 export KUBERNETES_NODE_MACHINE_CPU_REQUESTS="1"
 export KUBERNETES_NODE_MACHINE_MEMORY_REQUESTS="1Gi"
-clusterctl config cluster example \
-  --kubernetes-version=v1.17.0 \
+clusterctl config cluster $CLUSTER_NAME \
+  --kubernetes-version=v1.20.2 \
   --control-plane-machine-count=1 \
   --worker-machine-count=1 \
   | kubectl apply -f -
 
 # Retrieve kubeconfig
-until [ -n "`kubectl get secret example-kubeconfig -o jsonpath='{.data.value}' 2>/dev/null`" ] ; do
+until [ -n "`kubectl get secret $CLUSTER_NAME-kubeconfig -o jsonpath='{.data.value}' 2>/dev/null`" ] ; do
   sleep 1
 done
-kubectl get secret example-kubeconfig -o jsonpath='{.data.value}' | base64 --decode > example-kubeconfig
+kubectl get secret $CLUSTER_NAME-kubeconfig -o jsonpath='{.data.value}' | base64 --decode > $CLUSTER_NAME-kubeconfig
 
-# Switch to example cluster
+# Switch to new kind cluster
 # If the cluster api endpoint is not reachable from your machine you can exec into a
 # controller Node (Pod) and run `export KUBECONFIG=/etc/kubernetes/admin.conf` instead
-export KUBECONFIG=example-kubeconfig
+export KUBECONFIG=$CLUSTER_NAME-kubeconfig
 
 # Wait for the apiserver to come up
 until kubectl get nodes &>/dev/null; do
@@ -113,13 +120,16 @@ unset KUBECONFIG
 rm -f example-kubeconfig
 kubectl delete cluster example
 # If using the GKE example above
-yes | gcloud compute firewall-rules delete allow-management-cluster-pods-ipip
-yes | gcloud container clusters delete management --async
+yes | gcloud compute firewall-rules delete allow-$MANAGEMENT_CLUSTER_NAME-cluster-pods-ipip
+yes | gcloud container clusters delete $MANAGEMENT_CLUSTER_NAME --async
 ```
 
-[Cluster API]: https://github.com/kubernetes-sigs/cluster-api
-[Cluster API Infrastructure Provider]: https://cluster-api.sigs.k8s.io/reference/providers.html#infrastructure
-[kind]: https://github.com/kubernetes-sigs/kind
-[LoadBalancer Service]: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer
-[Calico]: https://docs.projectcalico.org/v3.11/getting-started/kubernetes/
-[IP-in-IP encapsulation]: https://docs.projectcalico.org/v3.11/getting-started/kubernetes/installation/config-options#configuring-ip-in-ip
+## TODO
+
+- Implement finalizer for control plane Pods to prevent deletion that'd lose quorum (i.e. PDB)
+- Work out why KCP replicas 3 has 0 failure tolerance
+  - https://github.com/kubernetes-sigs/cluster-api/blob/master/controlplane/kubeadm/controllers/remediation.go#L158-L159
+- Document how to configure persistence
+- Fix persistent control plane with 3 nodes
+- Default cluster service type to ClusterIP
+  - https://book.kubebuilder.io/cronjob-tutorial/webhook-implementation.html

api/v1alpha2/kubernetesmachine_conversion.go

@@ -27,6 +27,7 @@ func (src *KubernetesMachine) ConvertTo(dstRaw conversion.Hub) error {
 	dst.ObjectMeta = src.ObjectMeta
 
 	dst.Spec.ProviderID = src.Spec.ProviderID
+	dst.Spec.AllowRecreation = src.Spec.AllowRecreation
 	dst.Spec.PodSpec = src.Spec.PodSpec
 	dst.Spec.VolumeClaimTemplates = src.Spec.VolumeClaimTemplates
 
@@ -46,6 +47,7 @@ func (dst *KubernetesMachine) ConvertFrom(srcRaw conversion.Hub) error {
 	dst.ObjectMeta = src.ObjectMeta
 
 	dst.Spec.ProviderID = src.Spec.ProviderID
+	dst.Spec.AllowRecreation = src.Spec.AllowRecreation
 	dst.Spec.PodSpec = src.Spec.PodSpec
 	dst.Spec.VolumeClaimTemplates = src.Spec.VolumeClaimTemplates
 

api/v1alpha2/kubernetesmachine_types.go

@@ -31,10 +31,11 @@ const (
 
 // KubernetesMachineSpec defines the desired state of KubernetesMachine
 type KubernetesMachineSpec struct {
-	// ProviderID is in the form
-	// `kubernetes://<podUID>`.
+	// ProviderID is in the form `kubernetes://<namespace>/<name>`.
 	// +optional
 	ProviderID *string `json:"providerID,omitempty"`
+	// AllowRecreation allows machine Pod to be recreated
+	AllowRecreation bool `json:"allowRecreation,omitempty"`
 	// PodSpec forms the base of the Pod corresponding to the KubernetesMachine.
 	corev1.PodSpec `json:",inline"`
 	// VolumeClaimTemplates is a list of claims that PodSpec is allowed to

api/v1alpha2/kubernetesmachinetemplate_conversion.go

@@ -27,6 +27,7 @@ func (src *KubernetesMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	dst.ObjectMeta = src.ObjectMeta
 
 	dst.Spec.Template.Spec.ProviderID = src.Spec.Template.Spec.ProviderID
+	dst.Spec.Template.Spec.AllowRecreation = src.Spec.Template.Spec.AllowRecreation
 	dst.Spec.Template.Spec.PodSpec = src.Spec.Template.Spec.PodSpec
 	dst.Spec.Template.Spec.VolumeClaimTemplates = src.Spec.Template.Spec.VolumeClaimTemplates
 
@@ -40,6 +41,7 @@ func (dst *KubernetesMachineTemplate) ConvertFrom(srcRaw conversion.Hub) error {
 	dst.ObjectMeta = src.ObjectMeta
 
 	dst.Spec.Template.Spec.ProviderID = src.Spec.Template.Spec.ProviderID
+	dst.Spec.Template.Spec.AllowRecreation = src.Spec.Template.Spec.AllowRecreation
 	dst.Spec.Template.Spec.PodSpec = src.Spec.Template.Spec.PodSpec
 	dst.Spec.Template.Spec.VolumeClaimTemplates = src.Spec.Template.Spec.VolumeClaimTemplates
 

api/v1alpha3/kubernetesmachine_types.go

@@ -31,10 +31,11 @@ const (
 
 // KubernetesMachineSpec defines the desired state of KubernetesMachine
 type KubernetesMachineSpec struct {
-	// ProviderID is in the form
-	// `kubernetes://<podUID>`.
+	// ProviderID is in the form `kubernetes://<namespace>/<name>`.
 	// +optional
 	ProviderID *string `json:"providerID,omitempty"`
+	// AllowRecreation allows machine Pod to be recreated
+	AllowRecreation bool `json:"allowRecreation,omitempty"`
 	// PodSpec forms the base of the Pod corresponding to the KubernetesMachine.
 	corev1.PodSpec `json:",inline"`
 	// VolumeClaimTemplates is a list of claims that PodSpec is allowed to
@@ -57,13 +58,13 @@ type KubernetesMachineStatus struct {
 	// reconciling the KubernetesMachine and will contain a succinct value
 	// suitable for machine interpretation.
 	// +optional
-	FailureReason *capierrors.MachineStatusError `json:"errorReason,omitempty"`
+	FailureReason *capierrors.MachineStatusError `json:"failureReason,omitempty"`
 
 	// FailureMessage will be set in the event that there is a terminal problem
 	// reconciling the KubernetesMachine and will contain a more verbose string
 	// suitable for logging and human consumption.
 	// +optional
-	FailureMessage *string `json:"errorMessage,omitempty"`
+	FailureMessage *string `json:"failureMessage,omitempty"`
 
 	// Phase represents the current phase of KubernetesMachine actuation.
 	// +optional

config/crd/bases/infrastructure.lukeaddison.co.uk_kubernetesmachines.yaml

@@ -311,6 +311,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              allowRecreation:
+                type: boolean
               automountServiceAccountToken:
                 type: boolean
               containers:
@@ -3154,6 +3156,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              allowRecreation:
+                type: boolean
               automountServiceAccountToken:
                 type: boolean
               containers:
@@ -5688,9 +5692,9 @@ spec:
             type: object
           status:
             properties:
-              errorMessage:
+              failureMessage:
                 type: string
-              errorReason:
+              failureReason:
                 type: string
               phase:
                 type: string

config/crd/bases/infrastructure.lukeaddison.co.uk_kubernetesmachinetemplates.yaml

@@ -299,6 +299,8 @@ spec:
                                 type: array
                             type: object
                         type: object
+                      allowRecreation:
+                        type: boolean
                       automountServiceAccountToken:
                         type: boolean
                       containers:
@@ -3121,6 +3123,8 @@ spec:
                                 type: array
                             type: object
                         type: object
+                      allowRecreation:
+                        type: boolean
                       automountServiceAccountToken:
                         type: boolean
                       containers:

config/crd/kustomization.yaml

@@ -22,6 +22,45 @@ patchesStrategicMerge:
 - patches/cainjection_in_kubernetesmachinetemplates.yaml
 # +kubebuilder:scaffold:crdkustomizecainjectionpatch
 
+# Add spec.volumeClaimTemplates.metadata.name field to the OpenAPI v3.0 validation schema so that it
+# is preserved
+# https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#specifying-a-structural-schema
+# This could also be achieved using a patch like the following:
+# - op: add
+#   path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/volumeClaimTemplates/items/properties/metadata/x-kubernetes-preserve-unknown-fields
+#   value: true
+# but this would be less restrictive and the kubernetesmachine controller only makes use of the
+# spec.volumeClaimTemplates.metadata.name field
+patchesJSON6902:
+- target:
+    kind: CustomResourceDefinition
+    name: kubernetesmachines.infrastructure.lukeaddison.co.uk
+  patch: |-
+    - op: add
+      path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/volumeClaimTemplates/items/properties/metadata/properties
+      value:
+        name:
+          type: string
+    - op: add
+      path: /spec/versions/1/schema/openAPIV3Schema/properties/spec/properties/volumeClaimTemplates/items/properties/metadata/properties
+      value:
+        name:
+          type: string
+- target:
+    kind: CustomResourceDefinition
+    name: kubernetesmachinetemplates.infrastructure.lukeaddison.co.uk
+  patch: |-
+    - op: add
+      path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/volumeClaimTemplates/items/properties/metadata/properties
+      value:
+        name:
+          type: string
+    - op: add
+      path: /spec/versions/1/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/volumeClaimTemplates/items/properties/metadata/properties
+      value:
+        name:
+          type: string
+
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:
 - kustomizeconfig.yaml

controllers/kubernetescluster_controller.go

@@ -336,5 +336,5 @@ func (r *KubernetesClusterReconciler) createClusterService(ctx context.Context,
 }
 
 func clusterServiceName(cluster *clusterv1.Cluster) string {
-	return fmt.Sprintf("%s-lb", cluster.Name)
+	return fmt.Sprintf("%s-api-server", cluster.Name)
 }