Support Azure Stack dynamic environments

pkg/credentialprovider/azure/azure_credentials.go

@@ -144,7 +144,7 @@ func (a *acrProvider) loadConfig(rdr io.Reader) error {
 		klog.Errorf("Failed to load azure credential file: %v", err)
 	}
 
-	a.environment, err = auth.ParseAzureEnvironment(a.config.Cloud)
+	a.environment, err = auth.ParseAzureEnvironment(a.config.Cloud, a.config.CloudFQDN, a.config.IdentitySystem)
 	if err != nil {
 		return err
 	}

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go

@@ -30,11 +30,14 @@ import (
 	"k8s.io/klog"
 )
 
+const (
+	// ADFSIdentitySystem is the override value for tenantID on Azure Stack clouds.
+	ADFSIdentitySystem = "adfs"
+)
+
 var (
 	// ErrorNoAuth indicates that no credentials are provided.
 	ErrorNoAuth = fmt.Errorf("no credentials provided for Azure cloud provider")
-	// ADFSIdentitySystem indicates value of tenantId for ADFS on Azure Stack.
-	ADFSIdentitySystem = "ADFS"
 )
 
 // AzureAuthConfig holds auth related part of cloud config
@@ -59,15 +62,19 @@ type AzureAuthConfig struct {
 	UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"`
 	// The ID of the Azure Subscription that the cluster is deployed in
 	SubscriptionID string `json:"subscriptionId,omitempty" yaml:"subscriptionId,omitempty"`
-	// Identity system value for the deployment. This gets populate for Azure Stack case.
+	// IdentitySystem indicates the identity provider. Relevant only to hybrid clouds (Azure Stack).
+	// Allowed values are 'azure_ad' (default), 'adfs'.
 	IdentitySystem string `json:"identitySystem,omitempty" yaml:"identitySystem,omitempty"`
+	// CloudFQDN represents the hybrid cloud's fully qualified domain name: {location}.{domain}
+	// If set, cloud provider will generate its autorest.Environment instead of using one of the pre-defined ones.
+	CloudFQDN string `json:"cloudFQDN,omitempty" yaml:"cloudFQDN,omitempty"`
 }
 
 // GetServicePrincipalToken creates a new service principal token based on the configuration
 func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
 	var tenantID string
 	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
-		tenantID = "adfs"
+		tenantID = ADFSIdentitySystem
 	} else {
 		tenantID = config.TenantID
 	}
@@ -125,13 +132,25 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (
 	return nil, ErrorNoAuth
 }
 
-// ParseAzureEnvironment returns azure environment by name
-func ParseAzureEnvironment(cloudName string) (*azure.Environment, error) {
+// ParseAzureEnvironment returns the azure environment.
+// If 'cloudFQDN' is set, environment is computed by quering the cloud's resource manager endpoint.
+// Otherwise, a pre-defined Environment is looked up by name.
+func ParseAzureEnvironment(cloudName, cloudFQDN, identitySystem string) (*azure.Environment, error) {
 	var env azure.Environment
 	var err error
-	if cloudName == "" {
+	if cloudFQDN != "" {
+		resourceManagerEndpoint := fmt.Sprintf("https://management.%s/", cloudFQDN)
+		nameOverride := azure.OverrideProperty{Key: azure.EnvironmentName, Value: cloudName}
+		klog.V(4).Infof("Loading environment from resource manager endpoint: %s", resourceManagerEndpoint)
+		env, err = azure.EnvironmentFromURL(resourceManagerEndpoint, nameOverride)
+		if err == nil && strings.EqualFold(cloudName, "AzureStackCloud") {
+			azureStackOverrides(&env, cloudFQDN, identitySystem)
+		}
+	} else if cloudName == "" {
+		klog.V(4).Info("Using public cloud environment")
 		env = azure.PublicCloud
 	} else {
+		klog.V(4).Infof("Using %s environment", cloudName)
 		env, err = azure.EnvironmentFromName(cloudName)
 	}
 	return &env, err
@@ -151,3 +170,21 @@ func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.Private
 
 	return certificate, rsaPrivateKey, nil
 }
+
+// azureStackOverrides ensures that the Environment matches what AKSe currently generates for Azure Stack
+func azureStackOverrides(env *azure.Environment, cloudFQDN, identitySystem string) {
+	env.ManagementPortalURL = fmt.Sprintf("https://portal.%s/", cloudFQDN)
+	// TODO: figure out why AKSe does this
+	// why is autorest not setting ServiceManagementEndpoint?
+	env.ServiceManagementEndpoint = env.TokenAudience
+	// TODO: figure out why AKSe does this
+	// May not be required, ResourceManagerVMDNSSuffix is not used by k/k
+	split := strings.Split(cloudFQDN, ".")
+	domain := strings.Join(split[1:], ".")
+	env.ResourceManagerVMDNSSuffix = fmt.Sprintf("cloudapp.%s", domain)
+	// NOTE: autorest sets KeyVaultEndpoint while AKSe does not
+	if strings.EqualFold(identitySystem, ADFSIdentitySystem) {
+		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "/")
+		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "adfs")
+	}
+}

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

@@ -325,7 +325,7 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
 		}
 	}
 
-	env, err := auth.ParseAzureEnvironment(config.Cloud)
+	env, err := auth.ParseAzureEnvironment(config.Cloud, config.CloudFQDN, config.IdentitySystem)
 	if err != nil {
 		return err
 	}