Merge pull request #283 from snir911/peerpod-ctrl

peerpod ctrl

Dockerfile

@@ -20,7 +20,7 @@ RUN go mod download
 # needed for docker build but not for local builds
 RUN go mod vendor
 
-RUN GOFLAGS="" make build
+RUN make build
 
 # Use OpenShift base image
 FROM ${TARGET_IMAGE}

Makefile

@@ -70,6 +70,12 @@ TARGET_IMAGE  ?= registry.ci.openshift.org/ocp/4.13:base
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+BUILTIN_CLOUD_PROVIDERS ?= aws azure
+# Build tags required to build cloud-api-adaptor are derived from BUILTIN_CLOUD_PROVIDERS.
+space := $() $()
+comma := ,
+GOFLAGS := -tags=$(subst $(space),$(comma),$(strip $(BUILTIN_CLOUD_PROVIDERS)))
+
 .PHONY: all
 all: build
 
@@ -110,17 +116,15 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(GOFLAGS) ./... -coverprofile cover.out
 	# set write flag on created folder, so that we can clean it up
 	chmod +w $(LOCALBIN)/k8s/$(ENVTEST_K8S_VERSION)*
 
-
-
 ##@ Build
 
 .PHONY: build
 build: generate fmt vet ## Build manager binary.
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=mod -o bin/manager main.go
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GOFLAGS) -mod=mod -o bin/manager main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.

PROJECT

@@ -20,4 +20,12 @@ resources:
   webhooks:
     validation: true
     webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: confidentialcontainers.org
+  kind: PeerPod
+  path: github.com/confidential-containers/cloud-api-adaptor/peerpod-ctrl/api/v1alpha1
+  version: v1alpha1
 version: "3"

config/manager/manager.yaml

@@ -49,11 +49,28 @@ spec:
       - effect: NoSchedule
         key: node.kubernetes.io/memory-pressure
         operator: Exists
+      volumes:
+      - name: ssh
+        secret:
+          defaultMode: 384
+          optional: true
+          secretName: ssh-key-secret
       containers:
       - command:
         - /manager
         args:
         - --enable-leader-election
+        volumeMounts:
+        - mountPath: /root/.ssh/
+          name: ssh
+          readOnly: true
+        envFrom:
+        - secretRef:
+            name: peer-pods-secret
+            optional: true
+        - configMapRef:
+            name: peer-pods-cm
+            optional: true
         image: controller:latest
         name: manager
         env:

config/manifests/extension-crds/confidentialcontainers.org_peerpodconfigs.yaml

@@ -33,9 +33,6 @@ spec:
           metadata:
             type: object
           spec:
-            example:
-              type: string
-              x-kubernetes-preserve-unknown-fields: true
             description: PeerPodConfigSpec defines the desired state of PeerPodConfig
             properties:
               cloudSecretName:
@@ -52,57 +49,16 @@ spec:
                 description: InstanceType describes the name of the instance type
                   of the chosen cloud provider
                 type: string
-              labelSelector:
-                description: LabelSelector selects the nodes to which the caa pods,
-                  the RuntimeClass and the MachineConfigs we use to deploy the full
-                  peer pod solution.
-                properties:
-                  matchExpressions:
-                    description: matchExpressions is a list of label selector requirements.
-                      The requirements are ANDed.
-                    items:
-                      description: A label selector requirement is a selector that
-                        contains values, a key, and an operator that relates the key
-                        and values.
-                      properties:
-                        key:
-                          description: key is the label key that the selector applies
-                            to.
-                          type: string
-                        operator:
-                          description: operator represents a key's relationship to
-                            a set of values. Valid operators are In, NotIn, Exists
-                            and DoesNotExist.
-                          type: string
-                        values:
-                          description: values is an array of string values. If the
-                            operator is In or NotIn, the values array must be non-empty.
-                            If the operator is Exists or DoesNotExist, the values
-                            array must be empty. This array is replaced during a strategic
-                            merge patch.
-                          items:
-                            type: string
-                          type: array
-                      required:
-                      - key
-                      - operator
-                      type: object
-                    type: array
-                  matchLabels:
-                    additionalProperties:
-                      type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
-                      {key,value} in the matchLabels map is equivalent to an element
-                      of matchExpressions, whose key field is "key", the operator
-                      is "In", and the values array contains only "value". The requirements
-                      are ANDed.
-                    type: object
-                type: object
-                x-kubernetes-map-type: atomic
               limit:
                 description: Limit is the max number of peer pods. This is exposed
-                  as expended resource on nodes
+                  as extended resource on nodes
                 type: string
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: NodeSelector selects the nodes on which to run the cloud-api-adaptor
+                  pods
+                type: object
             required:
             - cloudSecretName
             - configMapName

config/manifests/extension-crds/confidentialcontainers.org_peerpods.yaml

@@ -0,0 +1,53 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.10.0
+  creationTimestamp: null
+  name: peerpods.confidentialcontainers.org
+spec:
+  group: confidentialcontainers.org
+  names:
+    kind: PeerPod
+    listKind: PeerPodList
+    plural: peerpods
+    singular: peerpod
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: PeerPod is the Schema for the peerpods API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: PeerPodSpec defines the desired state of PeerPod
+            properties:
+              cloudProvider:
+                type: string
+              instanceID:
+                type: string
+            type: object
+          status:
+            description: PeerPodStatus defines the observed state of PeerPod
+            properties:
+              cleand:
+                type: boolean
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/manifests/extension-crds/kustomization.yaml

@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - confidentialcontainers.org_peerpodconfigs.yaml
+- confidentialcontainers.org_peerpods.yaml

config/rbac/caa_rbac.yaml

@@ -0,0 +1,29 @@
+# This file is based on https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/install/rbac/peer-pod.yaml
+# It adds the required rules to the default SA which is used by CAA DA
+# when owner reference is both object must be on the same namespace, hence,
+# caa should have cluster-wide permissions to support any pod namespace
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: caa-role
+rules:
+- apiGroups: [""]
+  resources: ["pods", "pods/finalizers"]
+  verbs: ["get","create", "patch", "update"]
+- apiGroups: ["confidentialcontainers.org"]
+  resources: ["peerpods", "pods"]
+  verbs: ["create", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: caa-rolebinding
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: system
+roleRef:
+  kind: ClusterRole
+  name: caa-role
+  apiGroup: rbac.authorization.k8s.io

config/rbac/kataconfig_editor_role.yaml

@@ -7,9 +7,11 @@ rules:
 - apiGroups:
   - kataconfiguration.openshift.io
   - peerpodconfiguration.confidentialcontainers.org
+  - confidentialcontainers.org
   resources:
   - kataconfigs
   - peerpodconfigs
+  - peerpods
   verbs:
   - create
   - delete
@@ -21,8 +23,10 @@ rules:
 - apiGroups:
   - kataconfiguration.openshift.io
   - peerpodconfiguration.confidentialcontainers.org
+  - confidentialcontainers.org
   resources:
   - kataconfigs/status
   - peerpodconfigs/status
+  - peerpods/status
   verbs:
   - get

config/rbac/kataconfig_viewer_role.yaml

@@ -7,18 +7,22 @@ rules:
 - apiGroups:
   - kataconfiguration.openshift.io
   - peerpodconfiguration.confidentialcontainers.org
+  - confidentialcontainers.org
   resources:
   - kataconfigs
   - peerpodconfigs
+  - peerpods
   verbs:
   - get
   - list
   - watch
 - apiGroups:
   - peerpodconfiguration.confidentialcontainers.org
   - kataconfiguration.openshift.io
+  - confidentialcontainers.org
   resources:
   - kataconfigs/status
   - peerpodconfigs/status
+  - peerpods/status
   verbs:
   - get

config/rbac/kustomization.yaml

@@ -12,3 +12,6 @@ resources:
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+# the following is custom rbac manifests required for
+# cloud-api-adaptor when peerpod-ctrl is used
+- caa_rbac.yaml

config/samples/_v1alpha1_peerpod.yaml

@@ -0,0 +1,12 @@
+apiVersion: confidentialcontainers.org/v1alpha1
+kind: PeerPod
+metadata:
+  labels:
+    app.kubernetes.io/name: peerpod
+    app.kubernetes.io/instance: peerpod-sample
+    app.kubernetes.io/part-of: peerpod-ctrl
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: peerpod-ctrl
+  name: peerpod-sample
+spec:
+  # TODO(user): Add fields here

controllers/openshift_controller.go

@@ -95,6 +95,9 @@ const (
 // +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/finalizers,verbs=update
+// +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods/finalizers,verbs=update
 
 func (r *KataConfigOpenShiftReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	_ = r.Log.WithValues("kataconfig", req.NamespacedName)
@@ -1717,8 +1720,8 @@ func (r *KataConfigOpenShiftReconciler) updateFailedNodes(node *corev1.Node,
 	if err != nil {
 		return err, failedList
 	}
-	if (mcfgv1.IsMachineConfigPoolConditionTrue(foundMcp.Status.Conditions, mcfgv1.MachineConfigPoolNodeDegraded) ||
-		mcfgv1.IsMachineConfigPoolConditionTrue(foundMcp.Status.Conditions, mcfgv1.MachineConfigPoolDegraded)) {
+	if mcfgv1.IsMachineConfigPoolConditionTrue(foundMcp.Status.Conditions, mcfgv1.MachineConfigPoolNodeDegraded) ||
+		mcfgv1.IsMachineConfigPoolConditionTrue(foundMcp.Status.Conditions, mcfgv1.MachineConfigPoolDegraded) {
 		failedList =
 			append(r.kataConfig.Status.InstallationStatus.Failed.FailedNodesList,
 				kataconfigurationv1.FailedNodeStatus{Name: node.GetName(),