azure: don't use managed identity on ARO

At the moment OCP on Azure uses MSI for kubelets and controllers and one or
more service principals for operators.  For now on ARO, simplify to all
components using the user-provided SP.  Later, we'll reinstate a separate
managed identity at least for worker kubelets.

data/data/manifests/openshift/99_azure-cloud-provider-secret-getter-role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: system:azure-cloud-provider-secret-getter
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - azure-cloud-provider
+  resources:
+  - secrets
+  verbs:
+  - get

data/data/manifests/openshift/99_azure-cloud-provider-secret-getter-rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:azure-cloud-provider-secret-getter
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system:azure-cloud-provider-secret-getter
+subjects:
+- kind: ServiceAccount
+  name: azure-cloud-provider
+  namespace: kube-system

data/data/manifests/openshift/99_azure-cloud-provider-secret.yaml.template

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-cloud-provider
+  namespace: kube-system
+data:
+  cloud-config: {{.Base64encodedCloudConfig}}

pkg/asset/machines/azure/machines.go

@@ -101,7 +101,7 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		publicLB = ""
 	}
 
-	return &azureprovider.AzureMachineProviderSpec{
+	spec := &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
 			Kind:       "AzureMachineProviderSpec",
@@ -127,7 +127,13 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		ResourceGroup:        rg,
 		NetworkResourceGroup: networkResourceGroup,
 		PublicLoadBalancer:   publicLB,
-	}, nil
+	}
+
+	if platform.IsARO() {
+		spec.ManagedIdentity = ""
+	}
+
+	return spec, nil
 }
 
 // ConfigMasters sets the PublicIP flag and assigns a set of load balancers to the given machines

pkg/asset/manifests/azure/cloudproviderconfig.go

@@ -19,6 +19,7 @@ type CloudProviderConfig struct {
 	NetworkSecurityGroupName string
 	VirtualNetworkName       string
 	SubnetName               string
+	ARO                      bool
 }
 
 // JSON generates the cloud provider json config for the azure platform.
@@ -56,6 +57,11 @@ func (params CloudProviderConfig) JSON() (string, error) {
 		// https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-tcp-reset
 		LoadBalancerSku: "standard",
 	}
+
+	if params.ARO {
+		config.authConfig.UseManagedIdentityExtension = false
+	}
+
 	buff := &bytes.Buffer{}
 	encoder := json.NewEncoder(buff)
 	encoder.SetIndent("", "\t")

pkg/asset/manifests/cloudproviderconfig.go

@@ -134,6 +134,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 			NetworkSecurityGroupName: nsg,
 			VirtualNetworkName:       vnet,
 			SubnetName:               subnet,
+			ARO:                      installConfig.Config.Azure.IsARO(),
 		}.JSON()
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")

pkg/asset/manifests/openshift.go

@@ -69,6 +69,7 @@ func (o *Openshift) Dependencies() []asset.Asset {
 		&openshift.PrivateClusterOutbound{},
 		&openshift.BaremetalConfig{},
 		new(rhcos.Image),
+		&openshift.AzureCloudProviderSecret{},
 	}
 }
 
@@ -258,6 +259,38 @@ func (o *Openshift) Generate(dependencies asset.Parents) error {
 		assetData["99_private-cluster-outbound-service.yaml"] = applyTemplateData(privateClusterOutbound.Files()[0].Data, templateData)
 	}
 
+	if installConfig.Config.Azure.IsARO() {
+		// config is used to created compatible secret to trigger azure cloud
+		// controller config merge behaviour
+		// https://github.com/openshift/origin/blob/90c050f5afb4c52ace82b15e126efe98fa798d88/vendor/k8s.io/legacy-cloud-providers/azure/azure_config.go#L83
+		session, err := installConfig.Azure.Session()
+		if err != nil {
+			return err
+		}
+		config := struct {
+			AADClientID     string `json:"aadClientId" yaml:"aadClientId"`
+			AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+		}{
+			AADClientID:     session.Credentials.ClientID,
+			AADClientSecret: session.Credentials.ClientSecret,
+		}
+
+		b, err := yaml.Marshal(config)
+		if err != nil {
+			return err
+		}
+		base64encodedCloudConfig := base64.StdEncoding.EncodeToString(b)
+
+		azureCloudProviderSecret := &openshift.AzureCloudProviderSecret{}
+		dependencies.Get(azureCloudProviderSecret)
+		for _, f := range azureCloudProviderSecret.Files() {
+			name := strings.TrimSuffix(filepath.Base(f.Filename), ".template")
+			assetData[name] = applyTemplateData(f.Data, map[string]string{
+				"Base64encodedCloudConfig": base64encodedCloudConfig,
+			})
+		}
+	}
+
 	o.FileList = []*asset.File{}
 	for name, data := range assetData {
 		if len(data) == 0 {

pkg/asset/targets/targets.go

@@ -41,6 +41,7 @@ var (
 		&openshift.CloudCredsSecret{},
 		&openshift.KubeadminPasswordSecret{},
 		&openshift.RoleCloudCredsSecretReader{},
+		&openshift.AzureCloudProviderSecret{},
 	}
 
 	// IgnitionConfigs are the ignition-configs targeted assets.

pkg/asset/templates/content/openshift/azure-cloud-provider-secret.go

@@ -0,0 +1,80 @@
+package openshift
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/templates/content"
+)
+
+const (
+	azureCloudProviderSecretFileName                  = "99_azure-cloud-provider-secret.yaml.template"
+	azureCloudProviderSecretGetterRoleBindingFileName = "99_azure-cloud-provider-secret-getter-rolebinding.yaml"
+	azureCloudProviderSecretGetterRoleFileName        = "99_azure-cloud-provider-secret-getter-role.yaml"
+)
+
+var _ asset.WritableAsset = (*AzureCloudProviderSecret)(nil)
+
+// AzureCloudProviderSecret is the variable to represent contents of corresponding files
+type AzureCloudProviderSecret struct {
+	FileList []*asset.File
+}
+
+// Dependencies returns all of the dependencies directly needed by the asset
+func (t *AzureCloudProviderSecret) Dependencies() []asset.Asset {
+	return []asset.Asset{}
+}
+
+// Name returns the human-friendly name of the asset.
+func (t *AzureCloudProviderSecret) Name() string {
+	return "AzureCloudProviderSecret"
+}
+
+// Generate generates the actual files by this asset
+func (t *AzureCloudProviderSecret) Generate(parents asset.Parents) error {
+	t.FileList = []*asset.File{}
+
+	for _, fileName := range []string{
+		azureCloudProviderSecretFileName,
+		azureCloudProviderSecretGetterRoleBindingFileName,
+		azureCloudProviderSecretGetterRoleFileName,
+	} {
+		data, err := content.GetOpenshiftTemplate(fileName)
+		if err != nil {
+			return err
+		}
+		t.FileList = append(t.FileList, &asset.File{
+			Filename: filepath.Join(content.TemplateDir, fileName),
+			Data:     []byte(data),
+		})
+	}
+	return nil
+}
+
+// Files returns the files generated by the asset.
+func (t *AzureCloudProviderSecret) Files() []*asset.File {
+	return t.FileList
+}
+
+// Load returns the asset from disk.
+func (t *AzureCloudProviderSecret) Load(f asset.FileFetcher) (bool, error) {
+	t.FileList = []*asset.File{}
+
+	for _, fileName := range []string{
+		azureCloudProviderSecretFileName,
+		azureCloudProviderSecretGetterRoleBindingFileName,
+		azureCloudProviderSecretGetterRoleFileName,
+	} {
+		file, err := f.FetchByName(filepath.Join(content.TemplateDir, fileName))
+		if err != nil {
+			if os.IsNotExist(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		t.FileList = append(t.FileList, file)
+	}
+
+	return true, nil
+}