Skip to content

Commit

Permalink
add auditLogRoleARN field to ROSAControlPlane
Browse files Browse the repository at this point in the history
  • Loading branch information
@muraee
muraee committed Mar 12, 2024
1 parent 52469b2 commit 2b32d3d
Show file tree
Hide file tree
Showing 5 changed files with 203 additions and 131 deletions.
9 changes: 7 additions & 2 deletions config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ spec:
description: AdditionalTags are user-defined tags to be added on the
AWS resources associated with the control plane.
type: object
auditLogRoleARN:
description: AuditLogRoleARN defines the role that is used to forward
audit logs to AWS CloudWatch. If not set, audit log forwarding is
disabled.
type: string
availabilityZones:
description: AvailabilityZones describe AWS AvailabilityZones of the
worker nodes. should match the AvailabilityZones of the provided
Expand Down Expand Up @@ -146,8 +151,8 @@ spec:
- Public
- Private
type: string
etcdEncryptionKMSArn:
description: EtcdEncryptionKMSArn is the ARN of the KMS key used to
etcdEncryptionKMSARN:
description: EtcdEncryptionKMSARN is the ARN of the KMS key used to
encrypt etcd. The key itself needs to be created out-of-band by
the user and tagged with `red-hat:true`.
type: string
Expand Down
9 changes: 7 additions & 2 deletions controlplane/rosa/api/v1beta2/rosacontrolplane_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,15 @@ type RosaControlPlaneSpec struct { //nolint: maligned
// +optional
AdditionalTags infrav1.Tags `json:"additionalTags,omitempty"`

// EtcdEncryptionKMSArn is the ARN of the KMS key used to encrypt etcd. The key itself needs to be
// EtcdEncryptionKMSARN is the ARN of the KMS key used to encrypt etcd. The key itself needs to be
// created out-of-band by the user and tagged with `red-hat:true`.
// +optional
EtcdEncryptionKMSArn string `json:"etcdEncryptionKMSArn,omitempty"`
EtcdEncryptionKMSARN string `json:"etcdEncryptionKMSARN,omitempty"`

// AuditLogRoleARN defines the role that is used to forward audit logs to AWS CloudWatch.
// If not set, audit log forwarding is disabled.
// +optional
AuditLogRoleARN string `json:"auditLogRoleARN,omitempty"`

// CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API.
// The secret should contain the following data keys:
Expand Down
4 changes: 2 additions & 2 deletions controlplane/rosa/api/v1beta2/rosacontrolplane_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ func (r *ROSAControlPlane) validateNetwork() field.ErrorList {
}

func (r *ROSAControlPlane) validateEtcdEncryptionKMSArn() *field.Error {
err := kmsArnRegexpValidator.ValidateKMSKeyARN(&r.Spec.EtcdEncryptionKMSArn)
err := kmsArnRegexpValidator.ValidateKMSKeyARN(&r.Spec.EtcdEncryptionKMSARN)
if err != nil {
return field.Invalid(field.NewPath("spec.EtcdEncryptionKMSArn"), r.Spec.EtcdEncryptionKMSArn, err.Error())
return field.Invalid(field.NewPath("spec.etcdEncryptionKMSARN"), r.Spec.EtcdEncryptionKMSARN, err.Error())
}

return nil
Expand Down
278 changes: 157 additions & 121 deletions controlplane/rosa/controllers/rosacontrolplane_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,12 +241,16 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc
}
rosaScope.ControlPlane.Spec.ControlPlaneEndpoint = *apiEndpoint

if err := r.reconcileKubeconfig(ctx, rosaScope, ocmClient, cluster); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err)
if err := r.updateOCMCluster(rosaScope, ocmClient, cluster, creator); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to update rosa control plane: %w", err)
}
if err := r.reconcileClusterVersion(rosaScope, ocmClient, cluster); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcileKubeconfig(ctx, rosaScope, ocmClient, cluster); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err)
}

return ctrl.Result{}, nil
case cmv1.ClusterStateError:
errorMessage := cluster.Status().ProvisionErrorMessage()
Expand All @@ -272,80 +276,9 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc
return ctrl.Result{RequeueAfter: time.Second * 60}, nil
}

billingAccount := *rosaScope.Identity.Account
if rosaScope.ControlPlane.Spec.BillingAccount != "" {
billingAccount = rosaScope.ControlPlane.Spec.BillingAccount
}

ocmClusterSpec := ocm.Spec{
DryRun: ptr.To(false),
Name: rosaScope.RosaClusterName(),
DomainPrefix: rosaScope.ControlPlane.Spec.DomainPrefix,
Region: rosaScope.ControlPlane.Spec.Region,
MultiAZ: true,
Version: ocm.CreateVersionID(rosaScope.ControlPlane.Spec.Version, ocm.DefaultChannelGroup),
ChannelGroup: ocm.DefaultChannelGroup,
DisableWorkloadMonitoring: ptr.To(true),
DefaultIngress: ocm.NewDefaultIngressSpec(), // n.b. this is a no-op when it's set to the default value
ComputeMachineType: rosaScope.ControlPlane.Spec.DefaultMachinePoolSpec.InstanceType,
AvailabilityZones: rosaScope.ControlPlane.Spec.AvailabilityZones,
Tags: rosaScope.ControlPlane.Spec.AdditionalTags,
EtcdEncryption: rosaScope.ControlPlane.Spec.EtcdEncryptionKMSArn != "",
EtcdEncryptionKMSArn: rosaScope.ControlPlane.Spec.EtcdEncryptionKMSArn,

SubnetIds: rosaScope.ControlPlane.Spec.Subnets,
IsSTS: true,
RoleARN: rosaScope.ControlPlane.Spec.InstallerRoleARN,
SupportRoleARN: rosaScope.ControlPlane.Spec.SupportRoleARN,
WorkerRoleARN: rosaScope.ControlPlane.Spec.WorkerRoleARN,
OperatorIAMRoles: operatorIAMRoles(rosaScope.ControlPlane.Spec.RolesRef),
OidcConfigId: rosaScope.ControlPlane.Spec.OIDCID,
Mode: "auto",
Hypershift: ocm.Hypershift{
Enabled: true,
},
BillingAccount: billingAccount,
AWSCreator: creator,
}

if rosaScope.ControlPlane.Spec.EndpointAccess == rosacontrolplanev1.Private {
ocmClusterSpec.Private = ptr.To(true)
ocmClusterSpec.PrivateLink = ptr.To(true)
}

if networkSpec := rosaScope.ControlPlane.Spec.Network; networkSpec != nil {
if networkSpec.MachineCIDR != "" {
_, machineCIDR, err := net.ParseCIDR(networkSpec.MachineCIDR)
if err != nil {
return ctrl.Result{}, err
}
ocmClusterSpec.MachineCIDR = *machineCIDR
}

if networkSpec.PodCIDR != "" {
_, podCIDR, err := net.ParseCIDR(networkSpec.PodCIDR)
if err != nil {
return ctrl.Result{}, err
}
ocmClusterSpec.PodCIDR = *podCIDR
}

if networkSpec.ServiceCIDR != "" {
_, serviceCIDR, err := net.ParseCIDR(networkSpec.ServiceCIDR)
if err != nil {
return ctrl.Result{}, err
}
ocmClusterSpec.ServiceCIDR = *serviceCIDR
}

ocmClusterSpec.HostPrefix = networkSpec.HostPrefix
ocmClusterSpec.NetworkType = networkSpec.NetworkType
}

// Set cluster compute autoscaling replicas
if computeAutoscaling := rosaScope.ControlPlane.Spec.DefaultMachinePoolSpec.Autoscaling; computeAutoscaling != nil {
ocmClusterSpec.MaxReplicas = computeAutoscaling.MaxReplicas
ocmClusterSpec.MinReplicas = computeAutoscaling.MinReplicas
ocmClusterSpec, err := buildOCMClusterSpec(rosaScope.ControlPlane.Spec, creator)
if err != nil {
return ctrl.Result{}, err
}

cluster, err = ocmClient.CreateCluster(ocmClusterSpec)
Expand All @@ -364,51 +297,6 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc
return ctrl.Result{}, nil
}

func operatorIAMRoles(rolesRef rosacontrolplanev1.AWSRolesRef) []ocm.OperatorIAMRole {
return []ocm.OperatorIAMRole{
{
Name: "cloud-credentials",
Namespace: "openshift-ingress-operator",
RoleARN: rolesRef.IngressARN,
},
{
Name: "installer-cloud-credentials",
Namespace: "openshift-image-registry",
RoleARN: rolesRef.ImageRegistryARN,
},
{
Name: "ebs-cloud-credentials",
Namespace: "openshift-cluster-csi-drivers",
RoleARN: rolesRef.StorageARN,
},
{
Name: "cloud-credentials",
Namespace: "openshift-cloud-network-config-controller",
RoleARN: rolesRef.NetworkARN,
},
{
Name: "kube-controller-manager",
Namespace: "kube-system",
RoleARN: rolesRef.KubeCloudControllerARN,
},
{
Name: "kms-provider",
Namespace: "kube-system",
RoleARN: rolesRef.KMSProviderARN,
},
{
Name: "control-plane-operator",
Namespace: "kube-system",
RoleARN: rolesRef.ControlPlaneOperatorARN,
},
{
Name: "capa-controller-manager",
Namespace: "kube-system",
RoleARN: rolesRef.NodePoolManagementARN,
},
}
}

func (r *ROSAControlPlaneReconciler) reconcileDelete(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) {
rosaScope.Info("Reconciling ROSAControlPlane delete")

Expand Down Expand Up @@ -497,6 +385,29 @@ func (r *ROSAControlPlaneReconciler) reconcileClusterVersion(rosaScope *scope.RO
return nil
}

func (r *ROSAControlPlaneReconciler) updateOCMCluster(rosaScope *scope.ROSAControlPlaneScope, ocmClient *ocm.Client, cluster *cmv1.Cluster, creator *rosaaws.Creator) error {
currentAuditLogRole := cluster.AWS().AuditLog().RoleArn()
if currentAuditLogRole == rosaScope.ControlPlane.Spec.AuditLogRoleARN {
return nil
}

ocmClusterSpec := ocm.Spec{
AuditLogRoleARN: ptr.To(rosaScope.ControlPlane.Spec.AuditLogRoleARN),
}

// if this fails, the provided role is likely invalid or it doesn't have the required permissions.
if err := ocmClient.UpdateCluster(cluster.ID(), creator, ocmClusterSpec); err != nil {
conditions.MarkFalse(rosaScope.ControlPlane,
rosacontrolplanev1.ROSAControlPlaneValidCondition,
rosacontrolplanev1.ROSAControlPlaneInvalidConfigurationReason,
clusterv1.ConditionSeverityError,
err.Error())
return err
}

return nil
}

func (r *ROSAControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope, ocmClient *ocm.Client, cluster *cmv1.Cluster) error {
rosaScope.Debug("Reconciling ROSA kubeconfig for cluster", "cluster-name", rosaScope.RosaClusterName())

Expand Down Expand Up @@ -627,6 +538,131 @@ func validateControlPlaneSpec(ocmClient *ocm.Client, rosaScope *scope.ROSAContro
return "", nil
}

func buildOCMClusterSpec(controPlaneSpec rosacontrolplanev1.RosaControlPlaneSpec, creator *rosaaws.Creator) (ocm.Spec, error) {
billingAccount := controPlaneSpec.BillingAccount
if billingAccount == "" {
billingAccount = creator.AccountID
}

ocmClusterSpec := ocm.Spec{
DryRun: ptr.To(false),
Name: controPlaneSpec.RosaClusterName,
Region: controPlaneSpec.Region,
MultiAZ: true,
Version: ocm.CreateVersionID(controPlaneSpec.Version, ocm.DefaultChannelGroup),
ChannelGroup: ocm.DefaultChannelGroup,
DisableWorkloadMonitoring: ptr.To(true),
DefaultIngress: ocm.NewDefaultIngressSpec(), // n.b. this is a no-op when it's set to the default value
ComputeMachineType: controPlaneSpec.DefaultMachinePoolSpec.InstanceType,
AvailabilityZones: controPlaneSpec.AvailabilityZones,
Tags: controPlaneSpec.AdditionalTags,
EtcdEncryption: controPlaneSpec.EtcdEncryptionKMSARN != "",
EtcdEncryptionKMSArn: controPlaneSpec.EtcdEncryptionKMSARN,

SubnetIds: controPlaneSpec.Subnets,
IsSTS: true,
RoleARN: controPlaneSpec.InstallerRoleARN,
SupportRoleARN: controPlaneSpec.SupportRoleARN,
WorkerRoleARN: controPlaneSpec.WorkerRoleARN,
OperatorIAMRoles: operatorIAMRoles(controPlaneSpec.RolesRef),
OidcConfigId: controPlaneSpec.OIDCID,
Mode: "auto",
Hypershift: ocm.Hypershift{
Enabled: true,
},
BillingAccount: billingAccount,
AWSCreator: creator,
AuditLogRoleARN: ptr.To(controPlaneSpec.AuditLogRoleARN),
}

if controPlaneSpec.EndpointAccess == rosacontrolplanev1.Private {
ocmClusterSpec.Private = ptr.To(true)
ocmClusterSpec.PrivateLink = ptr.To(true)
}

if networkSpec := controPlaneSpec.Network; networkSpec != nil {
if networkSpec.MachineCIDR != "" {
_, machineCIDR, err := net.ParseCIDR(networkSpec.MachineCIDR)
if err != nil {
return ocmClusterSpec, err
}
ocmClusterSpec.MachineCIDR = *machineCIDR
}

if networkSpec.PodCIDR != "" {
_, podCIDR, err := net.ParseCIDR(networkSpec.PodCIDR)
if err != nil {
return ocmClusterSpec, err
}
ocmClusterSpec.PodCIDR = *podCIDR
}

if networkSpec.ServiceCIDR != "" {
_, serviceCIDR, err := net.ParseCIDR(networkSpec.ServiceCIDR)
if err != nil {
return ocmClusterSpec, err
}
ocmClusterSpec.ServiceCIDR = *serviceCIDR
}

ocmClusterSpec.HostPrefix = networkSpec.HostPrefix
ocmClusterSpec.NetworkType = networkSpec.NetworkType
}

// Set cluster compute autoscaling replicas
if computeAutoscaling := controPlaneSpec.DefaultMachinePoolSpec.Autoscaling; computeAutoscaling != nil {
ocmClusterSpec.MaxReplicas = computeAutoscaling.MaxReplicas
ocmClusterSpec.MinReplicas = computeAutoscaling.MinReplicas
}

return ocmClusterSpec, nil
}

func operatorIAMRoles(rolesRef rosacontrolplanev1.AWSRolesRef) []ocm.OperatorIAMRole {
return []ocm.OperatorIAMRole{
{
Name: "cloud-credentials",
Namespace: "openshift-ingress-operator",
RoleARN: rolesRef.IngressARN,
},
{
Name: "installer-cloud-credentials",
Namespace: "openshift-image-registry",
RoleARN: rolesRef.ImageRegistryARN,
},
{
Name: "ebs-cloud-credentials",
Namespace: "openshift-cluster-csi-drivers",
RoleARN: rolesRef.StorageARN,
},
{
Name: "cloud-credentials",
Namespace: "openshift-cloud-network-config-controller",
RoleARN: rolesRef.NetworkARN,
},
{
Name: "kube-controller-manager",
Namespace: "kube-system",
RoleARN: rolesRef.KubeCloudControllerARN,
},
{
Name: "kms-provider",
Namespace: "kube-system",
RoleARN: rolesRef.KMSProviderARN,
},
{
Name: "control-plane-operator",
Namespace: "kube-system",
RoleARN: rolesRef.ControlPlaneOperatorARN,
},
{
Name: "capa-controller-manager",
Namespace: "kube-system",
RoleARN: rolesRef.NodePoolManagementARN,
},
}
}

func (r *ROSAControlPlaneReconciler) rosaClusterToROSAControlPlane(log *logger.Logger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []ctrl.Request {
rosaCluster, ok := o.(*expinfrav1.ROSACluster)
Expand Down

0 comments on commit 2b32d3d

Please sign in to comment.