Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

feat: External cloud provider support for Azure Stack Cloud #4635

Merged
merged 8 commits into from Sep 3, 2021
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions parts/k8s/addons/azure-cloud-provider.yaml
Expand Up @@ -152,6 +152,7 @@ allowedTopologies:
{{else}}
volumeBindingMode: Immediate
{{- end}}
{{- if not IsAzureStackCloud}}
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
Expand All @@ -165,6 +166,7 @@ parameters:
reclaimPolicy: Delete
allowVolumeExpansion: true
volumeBindingMode: Immediate
{{- end}}
{{else}}
{{- if NeedsStorageAccountStorageClasses}}
---
Expand Down
60 changes: 59 additions & 1 deletion parts/k8s/addons/cloud-node-manager.yaml
Expand Up @@ -88,18 +88,50 @@ spec:
command:
- cloud-node-manager
- --node-name=$(NODE_NAME)
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=/etc/kubernetes/azure.json
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you know why only ASH needs cloud-config and kubeconfig?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For kubeconfig, if not provided the ecp (external cloud provider) will use default value from environment variable of Kubernetes Service (https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/vendor/k8s.io/client-go/rest/config.go#L488), which doesn't work for Azure Stack as well as Azure Windows node.

For cloud-config, we need that to correct get cloud and tenant information.

Copy link
Member

@jadarsie jadarsie Sep 3, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should figure out which KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT values work on ASH and Windows. The same for cloud and tenant info. The least access to the host file system, the better.

- --kubeconfig=/var/lib/kubelet/kubeconfig
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: DEBUG
{{end}}
resources:
requests:
cpu: 50m
memory: 50Mi
limits:
cpu: 2000m
memory: 512Mi
{{- if IsAzureStackCloud}}
volumeMounts:
- name: etc-kubernetes
mountPath: /etc/kubernetes
- name: etc-ssl
mountPath: /etc/ssl
readOnly: true
- name: var-lib-kubelet
mountPath: /var/lib/kubelet
readOnly: true
volumes:
- name: etc-kubernetes
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when possible, I would just mount the file instead of the entire directory:

volumes:
- name: etc-kubernetes
  hostPath:
    path: /etc/kubernetes/azurestackcloud.json
    type: FileOrCreate
volumeMounts:
- name: custom-environment
  mountPath: /etc/kubernetes/azurestackcloud.json
  readOnly: true

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The /etc/kubernetes directory also contains azure.json and cert, so I leave this one to folder level access. I updated the /var/lib/kubelet/kubeconfig to file level access.

hostPath:
path: /etc/kubernetes
- name: etc-ssl
hostPath:
path: /etc/ssl
- name: var-lib-kubelet
hostPath:
path: /var/lib/kubelet
{{end}}
{{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}}
---
apiVersion: apps/v1
Expand Down Expand Up @@ -148,16 +180,42 @@ spec:
command:
- /cloud-node-manager.exe
- --node-name=$(NODE_NAME)
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=C:\k\azure.json
- --kubeconfig=C:\k\config
lifecycle:
postStart:
exec:
command:
- C:\k\addazsroot.bat
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: C:\k\azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: DEBUG
{{end}}
resources:
requests:
cpu: 50m
memory: 50Mi
limits:
cpu: 2000m
memory: 512Mi
{{end}}
{{- if IsAzureStackCloud}}
volumeMounts:
- name: azure-config
mountPath: C:\k
volumes:
- name: azure-config
hostPath:
path: C:\k
type: Directory
{{end}}
{{end}}
30 changes: 29 additions & 1 deletion parts/k8s/kuberneteswindowssetup.ps1
Expand Up @@ -441,6 +441,34 @@ try
Register-NodeResetScriptTask
Update-DefenderPreferences

{{if IsAzureStackCloud}}
{{if UseCloudControllerManager}}
# Export the Azure Stack root cert for use in cloud node manager container setup.
$azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
if (Test-Path -Path $azsConfigFile) {
$azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
$azsARMUri = [System.Uri]$azsJson.managementPortalURL
$azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
if ($null -ne $azsRootCert) {
$azsRootCertFilePath = [io.path]::Combine($global:KubeDir, "azsroot.cer")
Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
haofan-ms marked this conversation as resolved.
Show resolved Hide resolved
}
}
}

# Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
$certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
if (Test-Path -Path $certocSourcePath) {
Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
}

# Create add cert script
$addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
[io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
haofan-ms marked this conversation as resolved.
Show resolved Hide resolved
{{end}}
{{end}}

if (Test-Path $CacheDir)
{
Write-Log "Removing aks-engine bits cache directory"
Expand Down Expand Up @@ -469,4 +497,4 @@ catch

Write-Error $_
throw $_
}
}
haofan-ms marked this conversation as resolved.
Show resolved Hide resolved
Expand Up @@ -13,6 +13,13 @@ spec:
- name: cloud-controller-manager
image: {{ContainerImage "cloud-controller-manager"}}
imagePullPolicy: IfNotPresent
{{- if IsAzureStackCloud}}
env:
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: DEBUG
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to keep this DEBUG verbosity for customers?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will update, thanks for catching this

{{end}}
command: [{{ContainerConfig "command"}}]
args: [{{GetCloudControllerManagerArgs}}]
resources:
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/addons.go
Expand Up @@ -581,7 +581,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {

defaultAzureFileCSIDriverAddonsConfig := KubernetesAddon{
Name: common.AzureFileCSIDriverAddonName,
Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName)),
Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName) && !cs.Properties.IsAzureStackCloud()),
Containers: []KubernetesContainerSpec{
{
Name: common.CSIProvisionerContainerName,
Expand Down Expand Up @@ -1176,7 +1176,7 @@ func getCSISidecarComponent(csiDriverName, csiSidecarName string, k8sComponents
// Otherwise, it returns empty string.
// Azure Stack needs the '-azs' suffix so kube-proxy's manifests uses the custom hyperkube image present in the VHD
func kubeProxyImageSuffix(cs ContainerService) string {
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return common.AzureStackSuffix
}
return ""
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/components.go
Expand Up @@ -307,7 +307,7 @@ func getComponentDefaultContainerImage(component string, cs *ContainerService) s

// componentImageSuffix returns '-azs' if target cloud is Azure Stack. Otherwise, it returns empty string.
func componentImageSuffix(cs ContainerService) string {
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return common.AzureStackSuffix
}
return ""
Expand Down
8 changes: 4 additions & 4 deletions pkg/api/k8s_versions.go
Expand Up @@ -45,8 +45,8 @@ const (
csiSnapshotControllerImageReference string = "oss/kubernetes-csi/snapshot-controller:v2.0.0"
csiAzureDiskImageReference string = "k8s/csi/azuredisk-csi:v0.7.0"
csiAzureFileImageReference string = "k8s/csi/azurefile-csi:v0.6.0"
azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v0.5.1"
azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v0.5.1"
azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.0"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it OK to update this for all clouds?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The change works for Azure and Azure Stack. For other cloud, I think they can continue to use with "useCloudControllerManager" flag set to false, or adopt similar change we did if they want to use external cloud provider as well. Does this make sense?

azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v1.1.0"
dashboardImageReference string = "mcr.microsoft.com/oss/kubernetes/dashboard:v2.0.4" // deprecated
dashboardMetricsScraperImageReference string = "mcr.microsoft.com/oss/kubernetes/metrics-scraper:v1.0.4"
kubeFlannelImageReference string = "quay.io/coreos/flannel:v0.8.0-amd64"
Expand Down Expand Up @@ -571,7 +571,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference,
common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference,
common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.DashboardAddonName: dashboardImageReference,
common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference,
common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),
Expand Down Expand Up @@ -655,7 +655,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference,
common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference,
common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.DashboardAddonName: dashboardImageReference,
common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference,
common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/types.go
Expand Up @@ -1919,7 +1919,7 @@ func (p *Properties) GetCustomCloudSourcesList() string {

// GetKubernetesVersion returns the cluster Kubernetes version, with the Azure Stack suffix if Azure Stack Cloud.
func (p *Properties) GetKubernetesVersion() string {
if p.IsAzureStackCloud() {
if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return p.OrchestratorProfile.OrchestratorVersion + AzureStackSuffix
}
return p.OrchestratorProfile.OrchestratorVersion
Expand All @@ -1930,7 +1930,7 @@ func (p *Properties) GetKubernetesHyperkubeSpec() string {
var kubernetesHyperkubeSpec string
k8sComponents := GetK8sComponentsByVersionMap(p.OrchestratorProfile.KubernetesConfig)[p.OrchestratorProfile.OrchestratorVersion]
kubernetesHyperkubeSpec = p.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + k8sComponents["hyperkube"]
if p.IsAzureStackCloud() {
if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
kubernetesHyperkubeSpec = kubernetesHyperkubeSpec + AzureStackSuffix
}
if p.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
Expand Down
2 changes: 1 addition & 1 deletion pkg/engine/template_generator.go
Expand Up @@ -698,7 +698,7 @@ version = 2
hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
k8sComponents := api.GetK8sComponentsByVersionMap(cs.Properties.OrchestratorProfile.KubernetesConfig)[cs.Properties.OrchestratorProfile.OrchestratorVersion]
hyperkubeImage := hyperkubeImageBase + k8sComponents[common.Hyperkube]
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
hyperkubeImage = hyperkubeImage + common.AzureStackSuffix
}
if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
Expand Down