Implement the NestedControllerManager controller

PROJECT

@@ -17,4 +17,9 @@ resources:
   group: controlplane
   kind: NestedAPIServer
   version: v1alpha4
+- api:
+    crdVersion: v1
+  group: controlplane
+  kind: NestedControllerManager
+  version: v1alpha4
 version: 3-alpha

apis/controlplane/v1alpha4/nestedcontrollermanager_types.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha4
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1"
+)
+
+// NestedControllerManagerSpec defines the desired state of NestedControllerManager
+type NestedControllerManagerSpec struct {
+	// NestedComponentSpec contains the common and user-specified information
+	// that are required for creating the component
+	// +optional
+	NestedComponentSpec `json:",inline"`
+}
+
+// NestedControllerManagerStatus defines the observed state of NestedControllerManager
+type NestedControllerManagerStatus struct {
+	// CommonStatus allows addons status monitoring
+	addonv1alpha1.CommonStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// NestedControllerManager is the Schema for the nestedcontrollermanagers API
+type NestedControllerManager struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   NestedControllerManagerSpec   `json:"spec,omitempty"`
+	Status NestedControllerManagerStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// NestedControllerManagerList contains a list of NestedControllerManager
+type NestedControllerManagerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []NestedControllerManager `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&NestedControllerManager{}, &NestedControllerManagerList{})
+}

config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml

@@ -8,7 +8,7 @@ spec:
   serviceName: {{.nestedAPIServerName}}
   selector:
     matchLabels:
-      component-name: apiserver
+      component-name: {{.nestedAPIServerName}}
   # apiserver will not be updated, unless it is deleted
   updateStrategy:
     type: OnDelete

config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{.nestedControllerManagerName}}
+  namespace: {{.nestedControllerManagerNamespace}}
+spec:
+  selector:
+    matchLabels:
+      component-name: {{.nestedControllerManagerName}}
+  updateStrategy:
+    type: OnDelete
+  template:
+    metadata:
+      labels:
+        component-name: {{.nestedControllerManagerName}}
+    spec:
+      containers:
+      - name: {{.nestedControllerManagerName}}
+        image: virtualcluster/controller-manager-v1.16.2
+        imagePullPolicy: Always
+        command:
+        - kube-controller-manager
+        args:
+        - --bind-address=0.0.0.0
+        - --cluster-cidr=10.200.0.0/16
+        - --cluster-signing-cert-file=/etc/kubernetes/pki/root/tls.crt
+        - --cluster-signing-key-file=/etc/kubernetes/pki/root/tls.key
+        - --kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
+        - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
+        - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
+        # control plane contains only one instance for now
+        - --leader-elect=false
+        - --root-ca-file=/etc/kubernetes/pki/root/tls.crt
+        - --service-account-private-key-file=/etc/kubernetes/pki/service-account/tls.key
+        - --service-cluster-ip-range=10.32.0.0/24
+        - --use-service-account-credentials=true
+        - --experimental-cluster-signing-duration=87600h
+        - --node-monitor-grace-period=200s
+        - --v=2
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10252
+            scheme: HTTP
+          failureThreshold: 8
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          timeoutSeconds: 15
+        readinessProbe:
+          httpGet:
+            port: 10252
+            path: /healthz
+            scheme: HTTP
+          failureThreshold: 8
+          initialDelaySeconds: 15
+          periodSeconds: 2
+          timeoutSeconds: 15
+        volumeMounts:
+        - mountPath: /etc/kubernetes/pki/root
+          name: {{.nestedControlPlaneName}}-apiserver
+          readOnly: true
+        - mountPath: /etc/kubernetes/pki/service-account
+          name: {{.nestedControlPlaneName}}-sa
+          readOnly: true
+        - mountPath: /etc/kubernetes/kubeconfig
+          name: {{.nestedControllerManagerName}}-kubeconfig
+          readOnly: true
+      volumes:
+      - name: {{.nestedControlPlaneName}}-apiserver
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-apiserver
+      - name: {{.nestedControlPlaneName}}-sa
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-sa
+      - name: {{.nestedControllerManagerName}}-kubeconfig
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControllerManagerName}}-kubeconfig

config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml

@@ -0,0 +1,96 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1-0.20201002000720-57250aac17f6
+  creationTimestamp: null
+  name: nestedcontrollermanagers.controlplane.cluster.x-k8s.io
+spec:
+  group: controlplane.cluster.x-k8s.io
+  names:
+    kind: NestedControllerManager
+    listKind: NestedControllerManagerList
+    plural: nestedcontrollermanagers
+    singular: nestedcontrollermanager
+  scope: Namespaced
+  versions:
+  - name: v1alpha4
+    schema:
+      openAPIV3Schema:
+        description: NestedControllerManager is the Schema for the nestedcontrollermanagers API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: NestedControllerManagerSpec defines the desired state of NestedControllerManager
+            properties:
+              channel:
+                description: 'Channel specifies a channel that can be used to resolve a specific addon, eg: stable It will be ignored if Version is specified'
+                type: string
+              patches:
+                items:
+                  type: object
+                type: array
+              replicas:
+                description: Replicas defines the number of replicas in the component's workload
+                format: int32
+                type: integer
+              resources:
+                description: Resources defines the amount of computing resources that will be used by this component
+                properties:
+                  limits:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
+                    type: object
+                  requests:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
+                    type: object
+                type: object
+              version:
+                description: Version specifies the exact addon version to be deployed, eg 1.2.3 It should not be specified if Channel is specified
+                type: string
+            type: object
+          status:
+            description: NestedControllerManagerStatus defines the observed state of NestedControllerManager
+            properties:
+              errors:
+                items:
+                  type: string
+                type: array
+              healthy:
+                type: boolean
+              phase:
+                type: string
+            required:
+            - healthy
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml
 - bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml
 - bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml
+- bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
@@ -13,13 +14,15 @@ patchesStrategicMerge:
 #- patches/webhook_in_nestedcontrolplanes.yaml
 #- patches/webhook_in_nestedetcds.yaml
 #- patches/webhook_in_nestedapiservers.yaml
+#- patches/webhook_in_nestedcontrollermanagers.yaml
 # +kubebuilder:scaffold:crdkustomizewebhookpatch
 
 # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
 #- patches/cainjection_in_nestedcontrolplanes.yaml
 #- patches/cainjection_in_nestedetcds.yaml
 #- patches/cainjection_in_nestedapiservers.yaml
+#- patches/cainjection_in_nestedcontrollermanagers.yaml
 # +kubebuilder:scaffold:crdkustomizecainjectionpatch
 
 # the following config is for teaching kustomize how to do kustomization for CRDs.

config/crd/patches/cainjection_in_nestedcontrollermanagers.yaml

@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: nestedcontrollermanagers.controlplane.cluster.x-k8s.io