azure: don't use managed identity on ARO

At the moment OCP on Azure uses MSI for kubelets and controllers and one or
more service principals for operators.  For now on ARO, simplify to all
components using the user-provided SP.  Later, we'll reinstate a separate
managed identity at least for worker kubelets.

pkg/asset/machines/azure/machines.go

@@ -101,7 +101,7 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		publicLB = ""
 	}
 
-	return &azureprovider.AzureMachineProviderSpec{
+	spec := &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
 			Kind:       "AzureMachineProviderSpec",
@@ -127,7 +127,13 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		ResourceGroup:        rg,
 		NetworkResourceGroup: networkResourceGroup,
 		PublicLoadBalancer:   publicLB,
-	}, nil
+	}
+
+	if platform.IsARO() {
+		spec.ManagedIdentity = ""
+	}
+
+	return spec, nil
 }
 
 // ConfigMasters sets the PublicIP flag and assigns a set of load balancers to the given machines

pkg/asset/manifests/azure/cloudproviderconfig.go

@@ -19,6 +19,7 @@ type CloudProviderConfig struct {
 	NetworkSecurityGroupName string
 	VirtualNetworkName       string
 	SubnetName               string
+	ARO                      bool
 }
 
 // JSON generates the cloud provider json config for the azure platform.
@@ -56,6 +57,11 @@ func (params CloudProviderConfig) JSON() (string, error) {
 		// https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-tcp-reset
 		LoadBalancerSku: "standard",
 	}
+
+	if params.ARO {
+		config.authConfig.UseManagedIdentityExtension = false
+	}
+
 	buff := &bytes.Buffer{}
 	encoder := json.NewEncoder(buff)
 	encoder.SetIndent("", "\t")

pkg/asset/manifests/cloudproviderconfig.go

@@ -134,6 +134,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 			NetworkSecurityGroupName: nsg,
 			VirtualNetworkName:       vnet,
 			SubnetName:               subnet,
+			ARO:                      installConfig.Config.Azure.IsARO(),
 		}.JSON()
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")

pkg/asset/manifests/openshift.go

@@ -11,10 +11,14 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/gophercloud/utils/openstack/clientconfig"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	installconfigaws "github.com/openshift/installer/pkg/asset/installconfig/aws"
+	icazure "github.com/openshift/installer/pkg/asset/installconfig/azure"
 	"github.com/openshift/installer/pkg/asset/installconfig/gcp"
 	kubeconfig "github.com/openshift/installer/pkg/asset/installconfig/kubevirt"
 	"github.com/openshift/installer/pkg/asset/installconfig/ovirt"
@@ -42,6 +46,10 @@ const (
 
 var (
 	_ asset.WritableAsset = (*Openshift)(nil)
+
+	aroCloudProviderRoleFileName        = "99_aro-cloud-provider-secret-reader-role.yaml"
+	aroCloudProviderRoleBindingFileName = "99_aro-cloud-provider-secret-reader-rolebinding.yaml"
+	aroCloudProviderSecretFileName      = "99_aro-cloud-provider-secret.yaml"
 )
 
 // Openshift generates the dependent resource manifests for openShift (as against bootkube)
@@ -258,6 +266,38 @@ func (o *Openshift) Generate(dependencies asset.Parents) error {
 		assetData["99_private-cluster-outbound-service.yaml"] = applyTemplateData(privateClusterOutbound.Files()[0].Data, templateData)
 	}
 
+	if installConfig.Config.Azure.IsARO() {
+		session, err := installConfig.Azure.Session()
+		if err != nil {
+			return errors.Wrap(err, "could not get azure session")
+		}
+
+		for _, f := range []struct {
+			filename string
+			data     func(icazure.Credentials) ([]byte, error)
+		}{
+			{
+				filename: aroCloudProviderRoleFileName,
+				data:     aroRole,
+			},
+			{
+				filename: aroCloudProviderRoleBindingFileName,
+				data:     aroRoleBinding,
+			},
+			{
+				filename: aroCloudProviderSecretFileName,
+				data:     aroSecret,
+			},
+		} {
+			b, err := f.data(session.Credentials)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create %s manifest", f.filename)
+			}
+
+			assetData[f.filename] = b
+		}
+	}
+
 	o.FileList = []*asset.File{}
 	for name, data := range assetData {
 		if len(data) == 0 {
@@ -309,3 +349,81 @@ func (o *Openshift) Load(f asset.FileFetcher) (bool, error) {
 	asset.SortFiles(o.FileList)
 	return len(o.FileList) > 0, nil
 }
+
+func aroRole(icazure.Credentials) ([]byte, error) {
+	return yaml.Marshal(&rbacv1.Role{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Role",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aro-cloud-provider-secret-reader",
+			Namespace: "kube-system",
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				Verbs:         []string{"get"},
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{"azure-cloud-provider"},
+			},
+		},
+	})
+}
+
+func aroRoleBinding(icazure.Credentials) ([]byte, error) {
+	return yaml.Marshal(&rbacv1.RoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "RoleBinding",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aro-cloud-provider-secret-read",
+			Namespace: "kube-system",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      "azure-cloud-provider",
+				Namespace: "kube-system",
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "aro-cloud-provider-secret-reader",
+		},
+	})
+}
+
+func aroSecret(platformCreds icazure.Credentials) ([]byte, error) {
+	// config is used to created compatible secret to trigger azure cloud
+	// controller config merge behaviour
+	// https://github.com/openshift/origin/blob/90c050f5afb4c52ace82b15e126efe98fa798d88/vendor/k8s.io/legacy-cloud-providers/azure/azure_config.go#L83
+	config := struct {
+		AADClientID     string `json:"aadClientId" yaml:"aadClientId"`
+		AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	}{
+		AADClientID:     platformCreds.ClientID,
+		AADClientSecret: platformCreds.ClientSecret,
+	}
+
+	b, err := yaml.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return yaml.Marshal(&corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azure-cloud-provider",
+			Namespace: "kube-system",
+		},
+		Data: map[string][]byte{
+			"cloud-config": b,
+		},
+	})
+}