diff --git a/api/constants/constants.go b/api/constants/constants.go index 37c114e..ef31223 100644 --- a/api/constants/constants.go +++ b/api/constants/constants.go @@ -16,6 +16,8 @@ const ( // ManagedByLabel is used to indicate which controller manages the resource. ManagedByLabel = OpenMCPGroupName + "/managed-by" + // ManagedPurposeLabel is used to indicate the purpose of the resource. + ManagedPurposeLabel = OpenMCPGroupName + "/managed-purpose" // OnboardingNameLabel is used to store the name on the onboarding cluster of a resource. OnboardingNameLabel = OpenMCPGroupName + "/onboarding-name" diff --git a/api/core/v2alpha1/constants.go b/api/core/v2alpha1/constants.go index 45eda60..5a79499 100644 --- a/api/core/v2alpha1/constants.go +++ b/api/core/v2alpha1/constants.go @@ -8,9 +8,13 @@ const ( ) const ( - MCPNameLabel = GroupName + "/mcp-name" - MCPNamespaceLabel = GroupName + "/mcp-namespace" - OIDCProviderLabel = GroupName + "/oidc-provider" + MCPNameLabel = GroupName + "/mcp-name" + MCPNamespaceLabel = GroupName + "/mcp-namespace" + OIDCProviderLabel = GroupName + "/oidc-provider" + MCPPurposeOverrideLabel = GroupName + "/purpose" + + // ManagedPurposeMCPPurposeOverride is used as value for the managed purpose label. It must not be modified. + ManagedPurposeMCPPurposeOverride = "mcp-purpose-override" MCPFinalizer = GroupName + "/mcp" diff --git a/cmd/openmcp-operator/app/mcp/init.go b/cmd/openmcp-operator/app/mcp/init.go index 7783ca9..70240f4 100644 --- a/cmd/openmcp-operator/app/mcp/init.go +++ b/cmd/openmcp-operator/app/mcp/init.go @@ -7,13 +7,18 @@ import ( "time" "github.com/spf13/cobra" + admissionv1 "k8s.io/api/admissionregistration/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" crdutil "github.com/openmcp-project/controller-utils/pkg/crds" + "github.com/openmcp-project/controller-utils/pkg/resources" clustersv1alpha1 "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1" apiconst "github.com/openmcp-project/openmcp-operator/api/constants" + corev2alpha1 "github.com/openmcp-project/openmcp-operator/api/core/v2alpha1" "github.com/openmcp-project/openmcp-operator/api/crds" "github.com/openmcp-project/openmcp-operator/api/install" "github.com/openmcp-project/openmcp-operator/cmd/openmcp-operator/app/options" @@ -21,6 +26,9 @@ import ( "github.com/openmcp-project/openmcp-operator/lib/clusteraccess" ) +// currently hard-coded, can be made configurable in the future if needed +const MCPPurposeOverrideValidationPolicyName = "mcp-purpose-override-validation" + func NewInitCommand(po *options.PersistentOptions) *cobra.Command { opts := &InitOptions{ PersistentOptions: po, @@ -94,6 +102,11 @@ func (o *InitOptions) Run(ctx context.Context) error { Resources: []string{"customresourcedefinitions"}, Verbs: []string{"*"}, }, + { + APIGroups: []string{"admissionregistration.k8s.io"}, + Resources: []string{"validatingadmissionpolicies", "validatingadmissionpolicybindings"}, + Verbs: []string{"*"}, + }, }, }, }) @@ -111,6 +124,119 @@ func (o *InitOptions) Run(ctx context.Context) error { if err := crdManager.CreateOrUpdateCRDs(ctx, &o.Log); err != nil { return fmt.Errorf("error creating/updating CRDs: %w", err) } + + // ensure ValidatingAdmissionPolicy to prevent removal or changes to the MCP purpose override label + labelSelector := client.MatchingLabels{ + apiconst.ManagedByLabel: managedcontrolplane.ControllerName, + apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride, + } + evapbs := &admissionv1.ValidatingAdmissionPolicyBindingList{} + if err := onboardingCluster.Client().List(ctx, evapbs, labelSelector); err != nil { + return fmt.Errorf("error listing ValidatingAdmissionPolicyBindings: %w", err) + } + for _, evapb := range evapbs.Items { + if evapb.Name != MCPPurposeOverrideValidationPolicyName { + log.Info("Deleting existing ValidatingAdmissionPolicyBinding with architecture immutability purpose", "name", evapb.Name) + if err := onboardingCluster.Client().Delete(ctx, &evapb); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("error deleting ValidatingAdmissionPolicyBinding '%s': %w", evapb.Name, err) + } + } + } + evaps := &admissionv1.ValidatingAdmissionPolicyList{} + if err := onboardingCluster.Client().List(ctx, evaps, labelSelector); err != nil { + return fmt.Errorf("error listing ValidatingAdmissionPolicies: %w", err) + } + for _, evap := range evaps.Items { + if evap.Name != MCPPurposeOverrideValidationPolicyName { + log.Info("Deleting existing ValidatingAdmissionPolicy with architecture immutability purpose", "name", evap.Name) + if err := onboardingCluster.Client().Delete(ctx, &evap); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("error deleting ValidatingAdmissionPolicy '%s': %w", evap.Name, err) + } + } + } + log.Info("creating/updating ValidatingAdmissionPolicies to prevent undesired changes to the MCP purpose override label ...") + vapm := resources.NewValidatingAdmissionPolicyMutator(MCPPurposeOverrideValidationPolicyName, admissionv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: ptr.To(admissionv1.Fail), + MatchConstraints: &admissionv1.MatchResources{ + ResourceRules: []admissionv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ // match all resources, actual restriction happens in the binding + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*"}, + }, + }, + }, + }, + }, + Variables: []admissionv1.Variable{ + { + Name: "purposeOverrideLabel", + Expression: fmt.Sprintf(`(has(object.metadata.labels) && "%s" in object.metadata.labels) ? object.metadata.labels["%s"] : ""`, corev2alpha1.MCPPurposeOverrideLabel, corev2alpha1.MCPPurposeOverrideLabel), + }, + { + Name: "oldPurposeOverrideLabel", + Expression: fmt.Sprintf(`(oldObject != null && has(oldObject.metadata.labels) && "%s" in oldObject.metadata.labels) ? oldObject.metadata.labels["%s"] : ""`, corev2alpha1.MCPPurposeOverrideLabel, corev2alpha1.MCPPurposeOverrideLabel), + }, + }, + Validations: []admissionv1.Validation{ + { + Expression: `request.operation == "CREATE" || (variables.oldPurposeOverrideLabel == variables.purposeOverrideLabel)`, + Message: fmt.Sprintf(`The label "%s" is immutable, it cannot be added after creation and is not allowed to be changed or removed once set.`, corev2alpha1.MCPPurposeOverrideLabel), + }, + { + Expression: `(variables.purposeOverrideLabel == "") || variables.purposeOverrideLabel.contains("mcp")`, + Message: fmt.Sprintf(`The value of the label "%s" must contain "mcp".`, corev2alpha1.MCPPurposeOverrideLabel), + }, + }, + }) + vapm.MetadataMutator().WithLabels(map[string]string{ + apiconst.ManagedByLabel: managedcontrolplane.ControllerName, + apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride, + }) + if err := resources.CreateOrUpdateResource(ctx, onboardingCluster.Client(), vapm); err != nil { + return fmt.Errorf("error creating/updating ValidatingAdmissionPolicy for mcp purpose override validation: %w", err) + } + + vapbm := resources.NewValidatingAdmissionPolicyBindingMutator(MCPPurposeOverrideValidationPolicyName, admissionv1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: MCPPurposeOverrideValidationPolicyName, + ValidationActions: []admissionv1.ValidationAction{ + admissionv1.Deny, + }, + MatchResources: &admissionv1.MatchResources{ + ResourceRules: []admissionv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{corev2alpha1.GroupVersion.Group}, + APIVersions: []string{corev2alpha1.GroupVersion.Version}, + Resources: []string{ + "managedcontrolplanev2s", + }, + }, + }, + }, + }, + }, + }) + vapbm.MetadataMutator().WithLabels(map[string]string{ + apiconst.ManagedByLabel: managedcontrolplane.ControllerName, + apiconst.ManagedPurposeLabel: corev2alpha1.ManagedPurposeMCPPurposeOverride, + }) + if err := resources.CreateOrUpdateResource(ctx, onboardingCluster.Client(), vapbm); err != nil { + return fmt.Errorf("error creating/updating ValidatingAdmissionPolicyBinding for mcp purpose override validation: %w", err) + } + log.Info("ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding for mcp purpose override validation created/updated") + log.Info("Finished init command") return nil } diff --git a/docs/README.md b/docs/README.md index 01ec6c1..8197595 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [AccessRequest Controller](controller/accessrequest.md) - [Deployment Controllers](controller/deployment.md) +- [ManagedControlPlane v2](controller/managedcontrolplane.md) - [Cluster Scheduler](controller/scheduler.md) ## Resources diff --git a/docs/controller/managedcontrolplane.md b/docs/controller/managedcontrolplane.md new file mode 100644 index 0000000..9b427c0 --- /dev/null +++ b/docs/controller/managedcontrolplane.md @@ -0,0 +1,82 @@ +# ManagedControlPlane v2 + +The *ManagedControlPlane v2 Controller* is a platform service that is responsible for reconciling `ManagedControlPlaneV2` (MCP) resources. + +Out of an MCP resource, it generates a `ClusterRequest` and multiple `AccessReqests`, thereby handling cluster management and authentication/authorization for MCPs. + +## Configuration + +The MCP controller takes the following configuration: +```yaml +managedControlPlane: + mcpClusterPurpose: mcp # defaults to 'mcp' + reconcileMCPEveryXDays: 7 # defaults to 0 + defaultOIDCProvider: + name: default # must be 'default' or omitted for the default oidc provider + issuer: https://oidc.example.com + clientID: my-client-id + usernamePrefix: "my-user:" + groupsPrefix: "my-group:" + extraScopes: + - foo +``` + +The configuration is optional. + +## ManagedControlPlaneV2 + +This is an example MCP resource: +```yaml +apiVersion: core.openmcp.cloud/v2alpha1 +kind: ManagedControlPlaneV2 +metadata: + name: mcp-01 + namespace: foo +spec: + iam: + roleBindings: # this sets the role bindings for the default OIDC provider (no effect if none is configured) + - subjects: + - kind: User + name: john.doe@example.com + roleRefs: + - kind: ClusterRole + name: cluster-admin + oidcProviders: # here, additional OIDC providers can be configured + - name: my-oidc-provider + issuer: https://oidc.example.com + clientID: my-client-id + usernamePrefix: "my-user:" + groupsPrefix: "my-group:" + extraScopes: + - foo + roleBindings: + - subjects: + - kind: User + name: foo + - kind: Group + name: bar + roleRefs: + - kind: ClusterRole + name: my-cluster-role + - kind: Role + name: my-role + namespace: default +``` + +### Purpose Overriding + +Usually, an MCP resource results in a `ClusterRequest` with its `spec.purpose` set to whatever is configured in the MCP controller configuration (defaults to `mcp` if not specified). The `core.openmcp.cloud/purpose` label allows to override this setting and specify a different purpose for a single MCP. + +Note that the purpose cannot be changed anymore after creation of the `ClusterRequest`, therefore the label has to be present already during creation of the MCP resource, it cannot be added afterwards. + +Also, it is not verified whether the chosen purpose actually is known to the scheduler. Specifying a unknown purpose will result in the MCP resource never becoming ready. + +#### Validation + +During setup, the MCP controller deploys a `ValidatingAdmissionPolicy` for the aforementioned label. It has the following effects: +- The label cannot be added or removed to/from an existing MCP resource. +- The label's value cannot be changed. +- The label's value must contain the substring `mcp`. + - This is meant to prevent customers (who have access to this label) from hijacking cluster purposes that are not meant for MCP clusters. + +This validation is currently not configurable in any way. diff --git a/internal/controllers/managedcontrolplane/controller.go b/internal/controllers/managedcontrolplane/controller.go index 4cc3c1d..eff3616 100644 --- a/internal/controllers/managedcontrolplane/controller.go +++ b/internal/controllers/managedcontrolplane/controller.go @@ -204,6 +204,12 @@ func (r *ManagedControlPlaneReconciler) handleCreateOrUpdate(ctx context.Context cr := &clustersv1alpha1.ClusterRequest{} cr.Name = mcp.Name cr.Namespace = platformNamespace + // determine cluster request purpose + purpose := r.Config.MCPClusterPurpose + if override, ok := mcp.Labels[corev2alpha1.MCPPurposeOverrideLabel]; ok && override != "" { + log.Info("Using purpose override from MCP label", "purposeOverride", override) + purpose = override + } if err := r.PlatformCluster.Client().Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { if !apierrors.IsNotFound(err) { rr.ReconcileError = errutils.WithReason(fmt.Errorf("unable to get ClusterRequest '%s/%s': %w", cr.Namespace, cr.Name, err), cconst.ReasonPlatformClusterInteractionProblem) @@ -211,10 +217,10 @@ func (r *ManagedControlPlaneReconciler) handleCreateOrUpdate(ctx context.Context return rr } - log.Info("ClusterRequest not found, creating it", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purpose", r.Config.MCPClusterPurpose) + log.Info("ClusterRequest not found, creating it", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purpose", purpose) cr.Labels = mcpLabels cr.Spec = clustersv1alpha1.ClusterRequestSpec{ - Purpose: r.Config.MCPClusterPurpose, + Purpose: purpose, WaitForClusterDeletion: ptr.To(true), } if err := r.PlatformCluster.Client().Create(ctx, cr); err != nil { @@ -223,7 +229,7 @@ func (r *ManagedControlPlaneReconciler) handleCreateOrUpdate(ctx context.Context return rr } } else { - log.Debug("ClusterRequest found", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "purposeInConfig", r.Config.MCPClusterPurpose, "purposeInClusterRequest", cr.Spec.Purpose) + log.Debug("ClusterRequest found", "clusterRequestName", cr.Name, "clusterRequestNamespace", cr.Namespace, "configuredPurpose", purpose, "purposeInClusterRequest", cr.Spec.Purpose) } // check if the ClusterRequest is ready diff --git a/internal/controllers/managedcontrolplane/controller_test.go b/internal/controllers/managedcontrolplane/controller_test.go index f6ffdb8..53772af 100644 --- a/internal/controllers/managedcontrolplane/controller_test.go +++ b/internal/controllers/managedcontrolplane/controller_test.go @@ -599,4 +599,24 @@ var _ = Describe("ManagedControlPlane Controller", func() { Expect(env.Client(platform).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(MatchError(apierrors.IsNotFound, "IsNotFound")) }) + It("should correctly set the purpose if the MCP has the override label", func() { + rec, env := defaultTestSetup("testdata", "test-01") + + mcp := &corev2alpha1.ManagedControlPlaneV2{} + mcp.SetName("mcp-02") + mcp.SetNamespace("test") + Expect(env.Client(onboarding).Get(env.Ctx, client.ObjectKeyFromObject(mcp), mcp)).To(Succeed()) + + platformNamespace, err := libutils.StableMCPNamespace(mcp.Name, mcp.Namespace) + Expect(err).ToNot(HaveOccurred()) + env.ShouldReconcile(mcpRec, testutils.RequestFromObject(mcp)) + cr := &clustersv1alpha1.ClusterRequest{} + cr.SetName(mcp.Name) + cr.SetNamespace(platformNamespace) + Expect(env.Client(platform).Get(env.Ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + Expect(cr.Spec.Purpose).ToNot(Equal(rec.Config.MCPClusterPurpose)) + Expect(cr.Spec.Purpose).To(Equal("my-mcp-purpose")) + Expect(cr.Spec.WaitForClusterDeletion).To(PointTo(BeTrue())) + }) + }) diff --git a/internal/controllers/managedcontrolplane/testdata/test-01/onboarding/mcp-02.yaml b/internal/controllers/managedcontrolplane/testdata/test-01/onboarding/mcp-02.yaml new file mode 100644 index 0000000..84ebff8 --- /dev/null +++ b/internal/controllers/managedcontrolplane/testdata/test-01/onboarding/mcp-02.yaml @@ -0,0 +1,8 @@ +apiVersion: core.openmcp.cloud/v2alpha1 +kind: ManagedControlPlaneV2 +metadata: + name: mcp-02 + namespace: test + labels: + core.openmcp.cloud/purpose: my-mcp-purpose +spec: {}