CARRY: HACK: don't use managed identity on ARO

At the moment OCP on Azure uses MSI for kubelets and controllers and one or
more service principals for operators.  For now on ARO, simplify to all
components using the user-provided SP.  Later, we'll reinstate a separate
managed identity at least for worker kubelets.

PR: openshift#4843

pkg/asset/machines/azure/machines.go

@@ -101,7 +101,7 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		publicLB = ""
 	}
 
-	return &azureprovider.AzureMachineProviderSpec{
+	spec := &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
 			Kind:       "AzureMachineProviderSpec",
@@ -127,7 +127,13 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		ResourceGroup:        rg,
 		NetworkResourceGroup: networkResourceGroup,
 		PublicLoadBalancer:   publicLB,
-	}, nil
+	}
+
+	if platform.ARO {
+		spec.ManagedIdentity = ""
+	}
+
+	return spec, nil
 }
 
 // ConfigMasters sets the PublicIP flag and assigns a set of load balancers to the given machines

pkg/asset/manifests/azure/cloudproviderconfig.go

@@ -19,6 +19,7 @@ type CloudProviderConfig struct {
 	NetworkSecurityGroupName string
 	VirtualNetworkName       string
 	SubnetName               string
+	ARO                      bool
 }
 
 // JSON generates the cloud provider json config for the azure platform.
@@ -56,6 +57,11 @@ func (params CloudProviderConfig) JSON() (string, error) {
 		// https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-tcp-reset
 		LoadBalancerSku: "standard",
 	}
+
+	if params.ARO {
+		config.authConfig.UseManagedIdentityExtension = false
+	}
+
 	buff := &bytes.Buffer{}
 	encoder := json.NewEncoder(buff)
 	encoder.SetIndent("", "\t")

pkg/asset/manifests/cloudproviderconfig.go

@@ -8,10 +8,13 @@ import (
 	"github.com/pkg/errors"
 
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	icazure "github.com/openshift/installer/pkg/asset/installconfig/azure"
 	"github.com/openshift/installer/pkg/asset/manifests/azure"
 	gcpmanifests "github.com/openshift/installer/pkg/asset/manifests/gcp"
 	kubevirtmanifests "github.com/openshift/installer/pkg/asset/manifests/kubevirt"
@@ -30,7 +33,10 @@ import (
 )
 
 var (
-	cloudProviderConfigFileName = filepath.Join(manifestDir, "cloud-provider-config.yaml")
+	cloudProviderConfigFileName         = filepath.Join(manifestDir, "cloud-provider-config.yaml")
+	aroCloudProviderRoleFileName        = filepath.Join(manifestDir, "aro-cloud-provider-secret-reader-role.yaml")
+	aroCloudProviderRoleBindingFileName = filepath.Join(manifestDir, "aro-cloud-provider-secret-reader-rolebinding.yaml")
+	aroCloudProviderSecretFileName      = filepath.Join(manifestDir, "aro-cloud-provider-secret.yaml")
 )
 
 const (
@@ -41,7 +47,7 @@ const (
 // CloudProviderConfig generates the cloud-provider-config.yaml files.
 type CloudProviderConfig struct {
 	ConfigMap *corev1.ConfigMap
-	File      *asset.File
+	FileList  []*asset.File
 }
 
 var _ asset.WritableAsset = (*CloudProviderConfig)(nil)
@@ -134,6 +140,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 			NetworkSecurityGroupName: nsg,
 			VirtualNetworkName:       vnet,
 			SubnetName:               subnet,
+			ARO:                      installConfig.Config.Azure.ARO,
 		}.JSON()
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")
@@ -181,22 +188,134 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 		return errors.Wrapf(err, "failed to create %s manifest", cpc.Name())
 	}
 	cpc.ConfigMap = cm
-	cpc.File = &asset.File{
-		Filename: cloudProviderConfigFileName,
-		Data:     cmData,
+	cpc.FileList = []*asset.File{
+		{
+			Filename: cloudProviderConfigFileName,
+			Data:     cmData,
+		},
+	}
+	if installConfig.Config.Azure.ARO {
+		session, err := installConfig.Azure.Session()
+		if err != nil {
+			return errors.Wrap(err, "could not get azure session")
+		}
+
+		for _, f := range []struct {
+			filename string
+			data     func(icazure.Credentials) ([]byte, error)
+		}{
+			{
+				filename: aroCloudProviderRoleFileName,
+				data:     aroRole,
+			},
+			{
+				filename: aroCloudProviderRoleBindingFileName,
+				data:     aroRoleBinding,
+			},
+			{
+				filename: aroCloudProviderSecretFileName,
+				data:     aroSecret,
+			},
+		} {
+			b, err := f.data(session.Credentials)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create %s manifest", cpc.Name())
+			}
+
+			cpc.FileList = append(cpc.FileList, &asset.File{
+				Filename: f.filename,
+				Data:     b,
+			})
+		}
 	}
 	return nil
 }
 
 // Files returns the files generated by the asset.
 func (cpc *CloudProviderConfig) Files() []*asset.File {
-	if cpc.File != nil {
-		return []*asset.File{cpc.File}
-	}
-	return []*asset.File{}
+	return cpc.FileList
 }
 
 // Load loads the already-rendered files back from disk.
 func (cpc *CloudProviderConfig) Load(f asset.FileFetcher) (bool, error) {
 	return false, nil
 }
+
+func aroRole(icazure.Credentials) ([]byte, error) {
+	return yaml.Marshal(&rbacv1.Role{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Role",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aro-cloud-provider-secret-reader",
+			Namespace: "kube-system",
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				Verbs:         []string{"get"},
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{"azure-cloud-provider"},
+			},
+		},
+	})
+}
+
+func aroRoleBinding(icazure.Credentials) ([]byte, error) {
+	return yaml.Marshal(&rbacv1.RoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "RoleBinding",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aro-cloud-provider-secret-read",
+			Namespace: "kube-system",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      "azure-cloud-provider",
+				Namespace: "kube-system",
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "aro-cloud-provider-secret-reader",
+		},
+	})
+}
+
+func aroSecret(platformCreds icazure.Credentials) ([]byte, error) {
+	// config is used to created compatible secret to trigger azure cloud
+	// controller config merge behaviour
+	// https://github.com/openshift/origin/blob/release-4.3/vendor/k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/azure/azure_config.go#L82
+	config := struct {
+		AADClientID     string `json:"aadClientId" yaml:"aadClientId"`
+		AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	}{
+		AADClientID:     platformCreds.ClientID,
+		AADClientSecret: platformCreds.ClientSecret,
+	}
+
+	b, err := yaml.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return yaml.Marshal(&v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azure-cloud-provider",
+			Namespace: "kube-system",
+		},
+		Data: map[string][]byte{
+			"cloud-config": b,
+		},
+		Type: v1.SecretTypeOpaque,
+	})
+}

pkg/asset/manifests/infrastructure.go

@@ -204,7 +204,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 	if cloudproviderconfig.ConfigMap != nil {
 		// set the configmap reference.
 		config.Spec.CloudConfig = configv1.ConfigMapFileReference{Name: cloudproviderconfig.ConfigMap.Name, Key: cloudProviderConfigDataKey}
-		i.FileList = append(i.FileList, cloudproviderconfig.File)
+		i.FileList = append(i.FileList, cloudproviderconfig.Files()...)
 	}
 
 	if trustbundleconfig.ConfigMap != nil {

pkg/types/azure/platform.go

@@ -75,6 +75,9 @@ type Platform struct {
 	//
 	// +optional
 	ResourceGroupName string `json:"resourceGroupName,omitempty"`
+
+	// ARO is a flag that indicates specialisations for the ARO platform
+	ARO bool `json:"aro,omitempty"`
 }
 
 // CloudEnvironment is the name of the Azure cloud environment