feat: multicluster support

.gitignore

@@ -16,7 +16,8 @@ install-config.yaml
 azure-env.sh
 .openshift*
 .DS_Store
-openshift-install
+openshift-install*
 node_modules
 .envrc
-.ansible/
+.ansible/
+__pycache__/

ansible/initdata-default.toml.tpl

@@ -25,3 +25,44 @@ kbs_cert = """
 {{ trustee_cert }}
 """
 '''
+
+"policy.rego" = '''
+package agent_policy
+
+default AddARPNeighborsRequest := true
+default AddSwapRequest := true
+default CloseStdinRequest := true
+default CopyFileRequest := true
+default CreateContainerRequest := true
+default CreateSandboxRequest := true
+default DestroySandboxRequest := true
+default ExecProcessRequest := false
+default GetMetricsRequest := true
+default GetOOMEventRequest := true
+default GuestDetailsRequest := true
+default ListInterfacesRequest := true
+default ListRoutesRequest := true
+default MemHotplugByProbeRequest := true
+default OnlineCPUMemRequest := true
+default PauseContainerRequest := true
+default PullImageRequest := true
+default ReadStreamRequest := false
+default RemoveContainerRequest := true
+default RemoveStaleVirtiofsShareMountsRequest := true
+default ReseedRandomDevRequest := true
+default ResumeContainerRequest := true
+default SetGuestDateTimeRequest := true
+default SetPolicyRequest := true
+default SignalProcessRequest := true
+default StartContainerRequest := true
+default StartTracingRequest := true
+default StatsContainerRequest := true
+default StopTracingRequest := true
+default TtyWinResizeRequest := true
+default UpdateContainerRequest := true
+default UpdateEphemeralMountsRequest := true
+default UpdateInterfaceRequest := true
+default UpdateRoutesRequest := true
+default WaitProcessRequest := true
+default WriteStreamRequest := true
+'''

charts/all/letsencrypt/values.yaml

@@ -3,6 +3,8 @@
 global:
   ## -- String containing the domain including the apps. prefix. Gets set by the Validated Pattern framework
   localClusterDomain: "apps.example.com"
+  ## -- String defining the cluster platform: "Azure" or "AWS" (overridden by values-global.yaml)
+  clusterPlatform: ""
 
 
 # -- This section contains all the parameters for the letsencrypt chart in
@@ -55,7 +57,7 @@ letsencrypt:
   azure:
     secretStoreKey: 'secret/data/global/azure'
 
-
+# Secret store configuration (overridden by values-global.yaml)
 secretStore:
-  name: vault-backend
-  kind: ClusterSecretStore
+  name: ""
+  kind: ""

charts/coco-supported/hello-openshift/templates/_helpers.tpl

@@ -0,0 +1,63 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "hello-openshift.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "hello-openshift.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hello-openshift.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "hello-openshift.labels" -}}
+helm.sh/chart: {{ include "hello-openshift.chart" . }}
+{{ include "hello-openshift.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "hello-openshift.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "hello-openshift.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Determine runtime class name based on cluster platform
+Returns "kata-remote" for Azure/AWS, "kata-cc" for other platforms
+*/}}
+{{- define "hello-openshift.runtimeClassName" -}}
+{{- if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}}
+kata-remote
+{{- else -}}
+kata-cc
+{{- end -}}
+{{- end }}

charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml

@@ -7,7 +7,7 @@ metadata:
   annotations:
     io.katacontainers.config.agent.policy: '{{ tpl ( .Files.Get "insecure-policy.rego") . | b64enc }}'
 spec:
-  runtimeClassName: kata-remote
+  runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }}
   containers:
     - name: hello-openshift
       image: quay.io/openshift/origin-hello-openshift

charts/coco-supported/hello-openshift/templates/secure-pod.yaml

@@ -7,7 +7,7 @@ metadata:
   annotations:
     peerpods: "true"
 spec:
-  runtimeClassName: kata-remote
+  runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }}
   containers:
     - name: hello-openshift
       image: quay.io/openshift/origin-hello-openshift

charts/coco-supported/hello-openshift/templates/standard-pod.yaml

@@ -5,7 +5,6 @@ metadata:
   labels:
     app: standard
 spec:
-  runtimeClassName: {{ .Values.global.runtimeClass }}
   containers:
     - name: hello-openshift
       image: quay.io/openshift/origin-hello-openshift

charts/coco-supported/hello-openshift/values.yaml

@@ -1,3 +1,6 @@
+# Chart-specific values
+# Common values are inherited from values-global.yaml
+
+# Global values used by this chart (overridden by values-global.yaml)
 global:
-  coco:
-    runtimeClassName: kata-remote
+  clusterPlatform: ""  # Cluster platform: "Azure" or "AWS" - determines runtime class

charts/coco-supported/kbs-access/values.yaml

@@ -1,3 +1,7 @@
+# Chart-specific values
+# Common values are inherited from values-global.yaml
+
+# Global values used by this chart (overridden by values-global.yaml)
 global:
   coco:
-    runtimeClassName: kata-remote
+    runtimeClassName: ""  # Runtime class for confidential containers

charts/coco-supported/sandbox/templates/kata-config.yaml

@@ -1,10 +1,8 @@
-{{ if .Values.sandbox.deploy }}
 apiVersion: kataconfiguration.openshift.io/v1
 kind: KataConfig
 metadata:
   annotations:
     argocd.argoproj.io/sync-wave: "100"
   name: default-kata-config
 spec:
-  enablePeerPods: true
-{{ end }}
+  enablePeerPods: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}true{{ else }}false{{ end }}

charts/coco-supported/sandbox/templates/ssh-key-eso.yaml

@@ -19,5 +19,4 @@ spec:
   dataFrom:
   - extract:
       key: {{ .Values.sandbox.sshKey }}
-
-{{ end }}
+{{- end }}

charts/coco-supported/sandbox/values.yaml

@@ -1,18 +1,18 @@
+# Chart-specific values
+# Common values are inherited from values-global.yaml
+
+# Global values used by this chart (overridden by values-global.yaml)
 global:
-  cocoUpstream: true
-  cocoConverged: true
+  clusterPlatform: ""  # Cluster platform: "Azure" or "AWS"
 
+# Secret store configuration (overridden by values-global.yaml)
 secretStore:
-  name: vault-backend
-  kind: ClusterSecretStore
-
-
+  name: ""
+  kind: ""
 
+# Sandbox-specific configuration
 sandbox:
-  deploy: true
   sshKey: secret/data/global/sshKey
-  azure: true
-  peerpodsCreds: secret/data/global/azure
   # These variables today limit to one cluster
   # revise using imperative framework to infer from cluster vars
   # Strongly advised to override in values-global.yaml or values-{cluster-group}.yaml

charts/hub/sandbox-policies/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+description: Deploy and configure policies for sandboxed containers and propagate data from the hub cluster to the managed clusters.
+keywords:
+- pattern
+name: sandbox-policies
+version: 0.0.1

charts/hub/sandbox-policies/README.md

@@ -0,0 +1,3 @@
+# Sandboxed policies
+
+Configure and propagate the policies, in particular the initdata, from the 'hub' cluster to the spoke cluster.

charts/hub/sandbox-policies/templates/hub-to-spoke-initdata-policy.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: Policy
+metadata:
+  name: hub-to-spoke-initdata-policy
+  namespace: imperative
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  remediationAction: enforce
+  disabled: false
+  policy-templates:
+  - objectDefinition:
+      apiVersion: policy.open-cluster-management.io/v1
+      kind: ConfigurationPolicy
+      metadata:
+        name: hub-to-spoke-initdata-cp
+        namespace: imperative
+      spec:
+        remediationAction: enforce
+        severity: medium
+        namespaceSelector:
+          include:
+            - imperative
+        object-templates:
+        - complianceType: mustonlyhave
+          objectDefinition:
+            apiVersion: v1
+            kind: ConfigMap
+            metadata:
+              name: initdata
+              namespace: imperative
+            data:
+              INITDATA: '{{ `{{hub fromConfigMap "imperative" "initdata" "INITDATA" hub}}` }}'
+
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: PlacementBinding
+metadata:
+  name: hub-to-spoke-initdata-placement-binding
+  namespace: imperative
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+placementRef:
+  name: hub-to-spoke-initdata-placement-rule
+  kind: PlacementRule
+  apiGroup: apps.open-cluster-management.io
+subjects:
+  - name: hub-to-spoke-initdata-policy
+    kind: Policy
+    apiGroup: policy.open-cluster-management.io
+
+---
+apiVersion: apps.open-cluster-management.io/v1
+kind: PlacementRule
+metadata:
+  name: hub-to-spoke-initdata-placement-rule
+  namespace: imperative
+spec:
+  clusterConditions:
+    - status: 'True'
+      type: ManagedClusterConditionAvailable
+  clusterSelector:
+    matchExpressions:
+      # Only apply to spoke clusters (exclude local-cluster which is typically the hub)
+      - key: name
+        operator: NotIn
+        values: ["local-cluster"]

charts/coco-supported/sandbox/templates/peer-pods-cm.yaml → charts/hub/sandbox-policies/templates/peer-pods-cm.yaml

@@ -1,4 +1,3 @@
-{{- if and .Values.sandbox.azure .Values.sandbox.deploy }}
 ---
 apiVersion: policy.open-cluster-management.io/v1
 kind: Policy
@@ -30,14 +29,14 @@ spec:
               VXLAN_PORT: "9000"
               AZURE_IMAGE_ID: '{{ `{{if (lookup "v1" "ConfigMap" "openshift-sandboxed-containers-operator" "peer-pods-cm").metadata.name }}{{ fromConfigMap "openshift-sandboxed-containers-operator" "peer-pods-cm" "AZURE_IMAGE_ID" }}{{ else }}{{ end }}` }}'
               AZURE_INSTANCE_SIZE: "{{ .Values.global.coco.azure.defaultVMFlavour }}" 
-              AZURE_INSTANCE_SIZES: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5"
+              AZURE_INSTANCE_SIZES: "{{ .Values.global.coco.azure.VMFlavours }}" 
               AZURE_RESOURCE_GROUP: '{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).vnetResourceGroup }}` }}'
               AZURE_REGION: '{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).location }}` }}'
               AZURE_SUBNET_ID: '/subscriptions/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).subscriptionId }}` }}/resourceGroups/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).vnetResourceGroup }}` }}/providers/Microsoft.Network/virtualNetworks/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).vnetName }}` }}/subnets/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).subnetName }}` }}'
               AZURE_NSG_ID: '/subscriptions/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).subscriptionId }}` }}/resourceGroups/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).resourceGroup }}` }}/providers/Microsoft.Network/networkSecurityGroups/{{ `{{ (fromJson (fromConfigMap "openshift-cloud-controller-manager" "cloud-conf" "cloud.conf" | toLiteral)).securityGroupName }}` }}'
               DISABLECVM: "false"
               PROXY_TIMEOUT: "5m"
-              INITDATA: '{{ `{{if (lookup "v1" "ConfigMap" "imperative" "initdata").metadata.name }}{{ fromConfigMap "imperative" "initdata" "INITDATA" }}{{ else }}{{ end }}` }}'
+              INITDATA: '{{ `{{ fromConfigMap "imperative" "initdata" "INITDATA" }}` }}'
 
 ---
 apiVersion: policy.open-cluster-management.io/v1
@@ -67,4 +66,3 @@ spec:
     matchLabels:
       cloud: Azure
 ---
-{{- end }}

charts/hub/sandbox-policies/values.yaml

@@ -0,0 +1,11 @@
+
+# Chart-specific values
+# Common values are inherited from values-global.yaml
+
+# Global values used by this chart (overridden by values-global.yaml)
+global:
+  clusterPlatform: ""  # Cluster platform: "Azure" or "AWS"
+  coco:
+    azure:
+      defaultVMFlavour: "Standard_DC2as_v5"
+      VMFlavours: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5"

charts/hub/trustee/templates/kbs-passphrase-eso.yaml → charts/hub/trustee/templates/dynamic-eso.yaml

@@ -1,23 +1,25 @@
 {{- if ne .Values.global.secretStore.backend "none" }}
+{{- range .Values.kbs.secretResources }}
 ---
 apiVersion: "external-secrets.io/v1beta1"
 kind: ExternalSecret
 metadata: 
   annotations:
     argocd.argoproj.io/sync-wave: "1"
-  name: kbs-passphrase-eso
-  namespace: {{ .Values.kbs.deployNS }}
+  name: {{ .name }}-eso
+  namespace: trustee-operator-system
 spec:
   refreshInterval: 15s
   secretStoreRef:
-    name: {{ .Values.secretStore.name }}
-    kind: {{ .Values.secretStore.kind }}
-  data:
+    name: {{ $.Values.secretStore.name }}
+    kind: {{ $.Values.secretStore.kind }}
   target:
-    name: passphrase
+    name: {{ .name }}
     template:
       type: Opaque
   dataFrom:
   - extract:
-      key: {{ .Values.kbs.passphrase }}
-{{- end }}
+      key: {{ .key }}
+{{- end }}
+{{- end }}
+