azure: don't use managed identity on ARO

data/data/manifests/openshift/99_azure-cloud-provider-secret-getter-role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: system:azure-cloud-provider-secret-getter
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - azure-cloud-provider
+  resources:
+  - secrets
+  verbs:
+  - get

data/data/manifests/openshift/99_azure-cloud-provider-secret-getter-rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:azure-cloud-provider-secret-getter
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system:azure-cloud-provider-secret-getter
+subjects:
+- kind: ServiceAccount
+  name: azure-cloud-provider
+  namespace: kube-system

data/data/manifests/openshift/99_azure-cloud-provider-secret.yaml.template

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-cloud-provider
+  namespace: kube-system
+stringData:
+  cloud-config: |
+    {{.CloudConfig | indent 4}}

pkg/asset/machines/azure/machines.go

@@ -101,6 +101,11 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		publicLB = ""
 	}
 
+	managedIdentity := fmt.Sprintf("%s-identity", clusterID)
+	if platform.IsARO() {
+		managedIdentity = ""
+	}
+
 	return &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
@@ -122,7 +127,7 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		},
 		Zone:                 az,
 		Subnet:               subnet,
-		ManagedIdentity:      fmt.Sprintf("%s-identity", clusterID),
+		ManagedIdentity:      managedIdentity,
 		Vnet:                 virtualNetwork,
 		ResourceGroup:        rg,
 		NetworkResourceGroup: networkResourceGroup,

pkg/asset/manifests/azure/cloudproviderconfig.go

@@ -19,17 +19,23 @@ type CloudProviderConfig struct {
 	NetworkSecurityGroupName string
 	VirtualNetworkName       string
 	SubnetName               string
+	ARO                      bool
 }
 
 // JSON generates the cloud provider json config for the azure platform.
 // managed resource names are matching the convention defined by capz
 func (params CloudProviderConfig) JSON() (string, error) {
+	useManagedIdentityExtension := true
+	if params.ARO {
+		useManagedIdentityExtension = false
+	}
+
 	config := config{
 		authConfig: authConfig{
 			Cloud:                       params.CloudName.Name(),
 			TenantID:                    params.TenantID,
 			SubscriptionID:              params.SubscriptionID,
-			UseManagedIdentityExtension: true,
+			UseManagedIdentityExtension: useManagedIdentityExtension,
 			// The cloud provider needs the clientID which is only known after terraform has run.
 			// When left empty, the existing managed identity on the VM will be used.
 			// By leaving it empty, we don't have to create the identity before running the installer.

pkg/asset/manifests/cloudproviderconfig.go

@@ -134,6 +134,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 			NetworkSecurityGroupName: nsg,
 			VirtualNetworkName:       vnet,
 			SubnetName:               subnet,
+			ARO:                      installConfig.Config.Azure.IsARO(),
 		}.JSON()
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")

pkg/asset/manifests/openshift.go

@@ -69,6 +69,7 @@ func (o *Openshift) Dependencies() []asset.Asset {
 		&openshift.PrivateClusterOutbound{},
 		&openshift.BaremetalConfig{},
 		new(rhcos.Image),
+		&openshift.AzureCloudProviderSecret{},
 	}
 }
 
@@ -258,6 +259,37 @@ func (o *Openshift) Generate(dependencies asset.Parents) error {
 		assetData["99_private-cluster-outbound-service.yaml"] = applyTemplateData(privateClusterOutbound.Files()[0].Data, templateData)
 	}
 
+	if platform == azuretypes.Name && installConfig.Config.Azure.IsARO() {
+		// config is used to created compatible secret to trigger azure cloud
+		// controller config merge behaviour
+		// https://github.com/openshift/origin/blob/90c050f5afb4c52ace82b15e126efe98fa798d88/vendor/k8s.io/legacy-cloud-providers/azure/azure_config.go#L83
+		session, err := installConfig.Azure.Session()
+		if err != nil {
+			return err
+		}
+		config := struct {
+			AADClientID     string `json:"aadClientId" yaml:"aadClientId"`
+			AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+		}{
+			AADClientID:     session.Credentials.ClientID,
+			AADClientSecret: session.Credentials.ClientSecret,
+		}
+
+		b, err := yaml.Marshal(config)
+		if err != nil {
+			return err
+		}
+
+		azureCloudProviderSecret := &openshift.AzureCloudProviderSecret{}
+		dependencies.Get(azureCloudProviderSecret)
+		for _, f := range azureCloudProviderSecret.Files() {
+			name := strings.TrimSuffix(filepath.Base(f.Filename), ".template")
+			assetData[name] = applyTemplateData(f.Data, map[string]string{
+				"CloudConfig": string(b),
+			})
+		}
+	}
+
 	o.FileList = []*asset.File{}
 	for name, data := range assetData {
 		if len(data) == 0 {

pkg/asset/targets/targets.go

@@ -41,6 +41,7 @@ var (
 		&openshift.CloudCredsSecret{},
 		&openshift.KubeadminPasswordSecret{},
 		&openshift.RoleCloudCredsSecretReader{},
+		&openshift.AzureCloudProviderSecret{},
 	}
 
 	// IgnitionConfigs are the ignition-configs targeted assets.

pkg/asset/templates/content/openshift/azure-cloud-provider-secret.go

@@ -0,0 +1,80 @@
+package openshift
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/templates/content"
+)
+
+const (
+	azureCloudProviderSecretFileName                  = "99_azure-cloud-provider-secret.yaml.template"
+	azureCloudProviderSecretGetterRoleBindingFileName = "99_azure-cloud-provider-secret-getter-rolebinding.yaml"
+	azureCloudProviderSecretGetterRoleFileName        = "99_azure-cloud-provider-secret-getter-role.yaml"
+)
+
+var _ asset.WritableAsset = (*AzureCloudProviderSecret)(nil)
+
+// AzureCloudProviderSecret is the variable to represent contents of corresponding files
+type AzureCloudProviderSecret struct {
+	FileList []*asset.File
+}
+
+// Dependencies returns all of the dependencies directly needed by the asset
+func (t *AzureCloudProviderSecret) Dependencies() []asset.Asset {
+	return []asset.Asset{}
+}
+
+// Name returns the human-friendly name of the asset.
+func (t *AzureCloudProviderSecret) Name() string {
+	return "AzureCloudProviderSecret"
+}
+
+// Generate generates the actual files by this asset
+func (t *AzureCloudProviderSecret) Generate(parents asset.Parents) error {
+	t.FileList = []*asset.File{}
+
+	for _, fileName := range []string{
+		azureCloudProviderSecretFileName,
+		azureCloudProviderSecretGetterRoleBindingFileName,
+		azureCloudProviderSecretGetterRoleFileName,
+	} {
+		data, err := content.GetOpenshiftTemplate(fileName)
+		if err != nil {
+			return err
+		}
+		t.FileList = append(t.FileList, &asset.File{
+			Filename: filepath.Join(content.TemplateDir, fileName),
+			Data:     []byte(data),
+		})
+	}
+	return nil
+}
+
+// Files returns the files generated by the asset.
+func (t *AzureCloudProviderSecret) Files() []*asset.File {
+	return t.FileList
+}
+
+// Load returns the asset from disk.
+func (t *AzureCloudProviderSecret) Load(f asset.FileFetcher) (bool, error) {
+	t.FileList = []*asset.File{}
+
+	for _, fileName := range []string{
+		azureCloudProviderSecretFileName,
+		azureCloudProviderSecretGetterRoleBindingFileName,
+		azureCloudProviderSecretGetterRoleFileName,
+	} {
+		file, err := f.FetchByName(filepath.Join(content.TemplateDir, fileName))
+		if err != nil {
+			if os.IsNotExist(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		t.FileList = append(t.FileList, file)
+	}
+
+	return true, nil
+}

pkg/types/azure/platform.go

@@ -5,6 +5,9 @@ import (
 	"strings"
 )
 
+// aro is a setting to enable aro-only modifications
+var aro bool
+
 // OutboundType is a strategy for how egress from cluster is achieved.
 // +kubebuilder:validation:Enum="";Loadbalancer;UserDefinedRouting
 type OutboundType string
@@ -115,3 +118,8 @@ func (p *Platform) ClusterResourceGroupName(infraID string) string {
 	}
 	return fmt.Sprintf("%s-rg", infraID)
 }
+
+// IsARO returns true if ARO-only modifications are enabled
+func (p *Platform) IsARO() bool {
+	return aro
+}

pkg/types/azure/platform_aro.go

@@ -0,0 +1,7 @@
+// +build aro
+
+package azure
+
+func init() {
+	aro = true
+}