Skip to content

Commit 36d0b56

Browse files
authored
feat: MCP purpose override (#151)
* implement mcp purpose override * fix logging and restrict label values to ones that contain 'mcp' * fix label mcp check * rename purpose override label * add test for purpose override * add documentation for MCPv2 * fix docs
1 parent 50157e8 commit 36d0b56

File tree

8 files changed

+255
-6
lines changed

8 files changed

+255
-6
lines changed

api/constants/constants.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ const (
1616

1717
// ManagedByLabel is used to indicate which controller manages the resource.
1818
ManagedByLabel = OpenMCPGroupName + "/managed-by"
19+
// ManagedPurposeLabel is used to indicate the purpose of the resource.
20+
ManagedPurposeLabel = OpenMCPGroupName + "/managed-purpose"
1921

2022
// OnboardingNameLabel is used to store the name on the onboarding cluster of a resource.
2123
OnboardingNameLabel = OpenMCPGroupName + "/onboarding-name"

api/core/v2alpha1/constants.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@ const (
88
)
99

1010
const (
11-
MCPNameLabel = GroupName + "/mcp-name"
12-
MCPNamespaceLabel = GroupName + "/mcp-namespace"
13-
OIDCProviderLabel = GroupName + "/oidc-provider"
11+
MCPNameLabel = GroupName + "/mcp-name"
12+
MCPNamespaceLabel = GroupName + "/mcp-namespace"
13+
OIDCProviderLabel = GroupName + "/oidc-provider"
14+
MCPPurposeOverrideLabel = GroupName + "/purpose"
15+
16+
// ManagedPurposeMCPPurposeOverride is used as value for the managed purpose label. It must not be modified.
17+
ManagedPurposeMCPPurposeOverride = "mcp-purpose-override"
1418

1519
MCPFinalizer = GroupName + "/mcp"
1620

cmd/openmcp-operator/app/mcp/init.go

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,28 @@ import (
77
"time"
88

99
"github.com/spf13/cobra"
10+
admissionv1 "k8s.io/api/admissionregistration/v1"
1011
rbacv1 "k8s.io/api/rbac/v1"
1112
"k8s.io/apimachinery/pkg/runtime"
13+
"k8s.io/utils/ptr"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
1215

1316
crdutil "github.com/openmcp-project/controller-utils/pkg/crds"
17+
"github.com/openmcp-project/controller-utils/pkg/resources"
1418

1519
clustersv1alpha1 "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1"
1620
apiconst "github.com/openmcp-project/openmcp-operator/api/constants"
21+
corev2alpha1 "github.com/openmcp-project/openmcp-operator/api/core/v2alpha1"
1722
"github.com/openmcp-project/openmcp-operator/api/crds"
1823
"github.com/openmcp-project/openmcp-operator/api/install"
1924
"github.com/openmcp-project/openmcp-operator/cmd/openmcp-operator/app/options"
2025
"github.com/openmcp-project/openmcp-operator/internal/controllers/managedcontrolplane"
2126
"github.com/openmcp-project/openmcp-operator/lib/clusteraccess"
2227
)
2328

29+
// currently hard-coded, can be made configurable in the future if needed
30+
const MCPPurposeOverrideValidationPolicyName = "mcp-purpose-override-validation"
31+
2432
func NewInitCommand(po *options.PersistentOptions) *cobra.Command {
2533
opts := &InitOptions{
2634
PersistentOptions: po,
@@ -94,6 +102,11 @@ func (o *InitOptions) Run(ctx context.Context) error {
94102
Resources: []string{"customresourcedefinitions"},
95103
Verbs: []string{"*"},
96104
},
105+
{
106+
APIGroups: []string{"admissionregistration.k8s.io"},
107+
Resources: []string{"validatingadmissionpolicies", "validatingadmissionpolicybindings"},
108+
Verbs: []string{"*"},
109+
},
97110
},
98111
},
99112
})
@@ -111,6 +124,119 @@ func (o *InitOptions) Run(ctx context.Context) error {
111124
if err := crdManager.CreateOrUpdateCRDs(ctx, &o.Log); err != nil {
112125
return fmt.Errorf("error creating/updating CRDs: %w", err)
113126
}
127+
128+
// ensure ValidatingAdmissionPolicy to prevent removal or changes to the MCP purpose override label
129+
labelSelector := client.MatchingLabels{
130+
apiconst.ManagedByLabel: managedcontrolplane.ControllerName,
131+
apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride,
132+
}
133+
evapbs := &admissionv1.ValidatingAdmissionPolicyBindingList{}
134+
if err := onboardingCluster.Client().List(ctx, evapbs, labelSelector); err != nil {
135+
return fmt.Errorf("error listing ValidatingAdmissionPolicyBindings: %w", err)
136+
}
137+
for _, evapb := range evapbs.Items {
138+
if evapb.Name != MCPPurposeOverrideValidationPolicyName {
139+
log.Info("Deleting existing ValidatingAdmissionPolicyBinding with architecture immutability purpose", "name", evapb.Name)
140+
if err := onboardingCluster.Client().Delete(ctx, &evapb); client.IgnoreNotFound(err) != nil {
141+
return fmt.Errorf("error deleting ValidatingAdmissionPolicyBinding '%s': %w", evapb.Name, err)
142+
}
143+
}
144+
}
145+
evaps := &admissionv1.ValidatingAdmissionPolicyList{}
146+
if err := onboardingCluster.Client().List(ctx, evaps, labelSelector); err != nil {
147+
return fmt.Errorf("error listing ValidatingAdmissionPolicies: %w", err)
148+
}
149+
for _, evap := range evaps.Items {
150+
if evap.Name != MCPPurposeOverrideValidationPolicyName {
151+
log.Info("Deleting existing ValidatingAdmissionPolicy with architecture immutability purpose", "name", evap.Name)
152+
if err := onboardingCluster.Client().Delete(ctx, &evap); client.IgnoreNotFound(err) != nil {
153+
return fmt.Errorf("error deleting ValidatingAdmissionPolicy '%s': %w", evap.Name, err)
154+
}
155+
}
156+
}
157+
log.Info("creating/updating ValidatingAdmissionPolicies to prevent undesired changes to the MCP purpose override label ...")
158+
vapm := resources.NewValidatingAdmissionPolicyMutator(MCPPurposeOverrideValidationPolicyName, admissionv1.ValidatingAdmissionPolicySpec{
159+
FailurePolicy: ptr.To(admissionv1.Fail),
160+
MatchConstraints: &admissionv1.MatchResources{
161+
ResourceRules: []admissionv1.NamedRuleWithOperations{
162+
{
163+
RuleWithOperations: admissionv1.RuleWithOperations{
164+
Operations: []admissionv1.OperationType{
165+
admissionv1.Create,
166+
admissionv1.Update,
167+
},
168+
Rule: admissionv1.Rule{ // match all resources, actual restriction happens in the binding
169+
APIGroups: []string{"*"},
170+
APIVersions: []string{"*"},
171+
Resources: []string{"*"},
172+
},
173+
},
174+
},
175+
},
176+
},
177+
Variables: []admissionv1.Variable{
178+
{
179+
Name: "purposeOverrideLabel",
180+
Expression: fmt.Sprintf(`(has(object.metadata.labels) && "%s" in object.metadata.labels) ? object.metadata.labels["%s"] : ""`, corev2alpha1.MCPPurposeOverrideLabel, corev2alpha1.MCPPurposeOverrideLabel),
181+
},
182+
{
183+
Name: "oldPurposeOverrideLabel",
184+
Expression: fmt.Sprintf(`(oldObject != null && has(oldObject.metadata.labels) && "%s" in oldObject.metadata.labels) ? oldObject.metadata.labels["%s"] : ""`, corev2alpha1.MCPPurposeOverrideLabel, corev2alpha1.MCPPurposeOverrideLabel),
185+
},
186+
},
187+
Validations: []admissionv1.Validation{
188+
{
189+
Expression: `request.operation == "CREATE" || (variables.oldPurposeOverrideLabel == variables.purposeOverrideLabel)`,
190+
Message: fmt.Sprintf(`The label "%s" is immutable, it cannot be added after creation and is not allowed to be changed or removed once set.`, corev2alpha1.MCPPurposeOverrideLabel),
191+
},
192+
{
193+
Expression: `(variables.purposeOverrideLabel == "") || variables.purposeOverrideLabel.contains("mcp")`,
194+
Message: fmt.Sprintf(`The value of the label "%s" must contain "mcp".`, corev2alpha1.MCPPurposeOverrideLabel),
195+
},
196+
},
197+
})
198+
vapm.MetadataMutator().WithLabels(map[string]string{
199+
apiconst.ManagedByLabel: managedcontrolplane.ControllerName,
200+
apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride,
201+
})
202+
if err := resources.CreateOrUpdateResource(ctx, onboardingCluster.Client(), vapm); err != nil {
203+
return fmt.Errorf("error creating/updating ValidatingAdmissionPolicy for mcp purpose override validation: %w", err)
204+
}
205+
206+
vapbm := resources.NewValidatingAdmissionPolicyBindingMutator(MCPPurposeOverrideValidationPolicyName, admissionv1.ValidatingAdmissionPolicyBindingSpec{
207+
PolicyName: MCPPurposeOverrideValidationPolicyName,
208+
ValidationActions: []admissionv1.ValidationAction{
209+
admissionv1.Deny,
210+
},
211+
MatchResources: &admissionv1.MatchResources{
212+
ResourceRules: []admissionv1.NamedRuleWithOperations{
213+
{
214+
RuleWithOperations: admissionv1.RuleWithOperations{
215+
Operations: []admissionv1.OperationType{
216+
admissionv1.Create,
217+
admissionv1.Update,
218+
},
219+
Rule: admissionv1.Rule{
220+
APIGroups: []string{corev2alpha1.GroupVersion.Group},
221+
APIVersions: []string{corev2alpha1.GroupVersion.Version},
222+
Resources: []string{
223+
"managedcontrolplanev2s",
224+
},
225+
},
226+
},
227+
},
228+
},
229+
},
230+
})
231+
vapbm.MetadataMutator().WithLabels(map[string]string{
232+
apiconst.ManagedByLabel: managedcontrolplane.ControllerName,
233+
apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride,
234+
})
235+
if err := resources.CreateOrUpdateResource(ctx, onboardingCluster.Client(), vapbm); err != nil {
236+
return fmt.Errorf("error creating/updating ValidatingAdmissionPolicyBinding for mcp purpose override validation: %w", err)
237+
}
238+
log.Info("ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding for mcp purpose override validation created/updated")
239+
114240
log.Info("Finished init command")
115241
return nil
116242
}

docs/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
- [AccessRequest Controller](controller/accessrequest.md)
77
- [Deployment Controllers](controller/deployment.md)
8+
- [ManagedControlPlane v2](controller/managedcontrolplane.md)
89
- [Cluster Scheduler](controller/scheduler.md)
910

1011
## Resources
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# ManagedControlPlane v2
2+
3+
The *ManagedControlPlane v2 Controller* is a platform service that is responsible for reconciling `ManagedControlPlaneV2` (MCP) resources.
4+
5+
Out of an MCP resource, it generates a `ClusterRequest` and multiple `AccessReqests`, thereby handling cluster management and authentication/authorization for MCPs.
6+
7+
## Configuration
8+
9+
The MCP controller takes the following configuration:
10+
```yaml
11+
managedControlPlane:
12+
mcpClusterPurpose: mcp # defaults to 'mcp'
13+
reconcileMCPEveryXDays: 7 # defaults to 0
14+
defaultOIDCProvider:
15+
name: default # must be 'default' or omitted for the default oidc provider
16+
issuer: https://oidc.example.com
17+
clientID: my-client-id
18+
usernamePrefix: "my-user:"
19+
groupsPrefix: "my-group:"
20+
extraScopes:
21+
- foo
22+
```
23+
24+
The configuration is optional.
25+
26+
## ManagedControlPlaneV2
27+
28+
This is an example MCP resource:
29+
```yaml
30+
apiVersion: core.openmcp.cloud/v2alpha1
31+
kind: ManagedControlPlaneV2
32+
metadata:
33+
name: mcp-01
34+
namespace: foo
35+
spec:
36+
iam:
37+
roleBindings: # this sets the role bindings for the default OIDC provider (no effect if none is configured)
38+
- subjects:
39+
- kind: User
40+
name: john.doe@example.com
41+
roleRefs:
42+
- kind: ClusterRole
43+
name: cluster-admin
44+
oidcProviders: # here, additional OIDC providers can be configured
45+
- name: my-oidc-provider
46+
issuer: https://oidc.example.com
47+
clientID: my-client-id
48+
usernamePrefix: "my-user:"
49+
groupsPrefix: "my-group:"
50+
extraScopes:
51+
- foo
52+
roleBindings:
53+
- subjects:
54+
- kind: User
55+
name: foo
56+
- kind: Group
57+
name: bar
58+
roleRefs:
59+
- kind: ClusterRole
60+
name: my-cluster-role
61+
- kind: Role
62+
name: my-role
63+
namespace: default
64+
```
65+
66+
### Purpose Overriding
67+
68+
Usually, an MCP resource results in a `ClusterRequest` with its `spec.purpose` set to whatever is configured in the MCP controller configuration (defaults to `mcp` if not specified). The `core.openmcp.cloud/purpose` label allows to override this setting and specify a different purpose for a single MCP.
69+
70+
Note that the purpose cannot be changed anymore after creation of the `ClusterRequest`, therefore the label has to be present already during creation of the MCP resource, it cannot be added afterwards.
71+
72+
Also, it is not verified whether the chosen purpose actually is known to the scheduler. Specifying a unknown purpose will result in the MCP resource never becoming ready.
73+
74+
#### Validation
75+
76+
During setup, the MCP controller deploys a `ValidatingAdmissionPolicy` for the aforementioned label. It has the following effects:
77+
- The label cannot be added or removed to/from an existing MCP resource.
78+
- The label's value cannot be changed.
79+
- The label's value must contain the substring `mcp`.
80+
- This is meant to prevent customers (who have access to this label) from hijacking cluster purposes that are not meant for MCP clusters.
81+
82+
This validation is currently not configurable in any way.

internal/controllers/managedcontrolplane/controller.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,17 +204,23 @@ func (r *ManagedControlPlaneReconciler) handleCreateOrUpdate(ctx context.Context
204204
cr := &clustersv1alpha1.ClusterRequest{}
205205
cr.Name = mcp.Name
206206
cr.Namespace = platformNamespace
207+
// determine cluster request purpose
208+
purpose := r.Config.MCPClusterPurpose
209+
if override, ok := mcp.Labels[corev2alpha1.MCPPurposeOverrideLabel]; ok && override != "" {
210+
log.Info("Using purpose override from MCP label", "purposeOverride", override)
211+
purpose = override
212+
}
207213
if err := r.PlatformCluster.Client().Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil {
208214
if !apierrors.IsNotFound(err) {
209215
rr.ReconcileError = errutils.WithReason(fmt.Errorf("unable to get ClusterRequest '%s/%s': %w", cr.Namespace, cr.Name, err), cconst.ReasonPlatformClusterInteractionProblem)
210216
createCon(corev2alpha1.ConditionClusterRequestReady, metav1.ConditionFalse, rr.ReconcileError.Reason(), rr.ReconcileError.Error())
211217
return rr
212218
}
213219

214-
log.Info("ClusterRequest not found, creating it", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purpose", r.Config.MCPClusterPurpose)
220+
log.Info("ClusterRequest not found, creating it", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purpose", purpose)
215221
cr.Labels = mcpLabels
216222
cr.Spec = clustersv1alpha1.ClusterRequestSpec{
217-
Purpose: r.Config.MCPClusterPurpose,
223+
Purpose: purpose,
218224
WaitForClusterDeletion: ptr.To(true),
219225
}
220226
if err := r.PlatformCluster.Client().Create(ctx, cr); err != nil {
@@ -223,7 +229,7 @@ func (r *ManagedControlPlaneReconciler) handleCreateOrUpdate(ctx context.Context
223229
return rr
224230
}
225231
} else {
226-
log.Debug("ClusterRequest found", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purposeInConfig", r.Config.MCPClusterPurpose, "purposeInClusterRequest", cr.Spec.Purpose)
232+
log.Debug("ClusterRequest found", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "configuredPurpose", purpose, "purposeInClusterRequest", cr.Spec.Purpose)
227233
}
228234

229235
// check if the ClusterRequest is ready

internal/controllers/managedcontrolplane/controller_test.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,4 +599,24 @@ var _ = Describe("ManagedControlPlane Controller", func() {
599599
Expect(env.Client(platform).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(MatchError(apierrors.IsNotFound, "IsNotFound"))
600600
})
601601

602+
It("should correctly set the purpose if the MCP has the override label", func() {
603+
rec, env := defaultTestSetup("testdata", "test-01")
604+
605+
mcp := &corev2alpha1.ManagedControlPlaneV2{}
606+
mcp.SetName("mcp-02")
607+
mcp.SetNamespace("test")
608+
Expect(env.Client(onboarding).Get(env.Ctx, client.ObjectKeyFromObject(mcp), mcp)).To(Succeed())
609+
610+
platformNamespace, err := libutils.StableMCPNamespace(mcp.Name, mcp.Namespace)
611+
Expect(err).ToNot(HaveOccurred())
612+
env.ShouldReconcile(mcpRec, testutils.RequestFromObject(mcp))
613+
cr := &clustersv1alpha1.ClusterRequest{}
614+
cr.SetName(mcp.Name)
615+
cr.SetNamespace(platformNamespace)
616+
Expect(env.Client(platform).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed())
617+
Expect(cr.Spec.Purpose).ToNot(Equal(rec.Config.MCPClusterPurpose))
618+
Expect(cr.Spec.Purpose).To(Equal("my-mcp-purpose"))
619+
Expect(cr.Spec.WaitForClusterDeletion).To(PointTo(BeTrue()))
620+
})
621+
602622
})
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: core.openmcp.cloud/v2alpha1
2+
kind: ManagedControlPlaneV2
3+
metadata:
4+
name: mcp-02
5+
namespace: test
6+
labels:
7+
core.openmcp.cloud/purpose: my-mcp-purpose
8+
spec: {}

0 commit comments

Comments
 (0)