diff --git a/parts/k8s/addons/azure-cloud-provider.yaml b/parts/k8s/addons/azure-cloud-provider.yaml index 7164c7b79c..8df9c8adcd 100644 --- a/parts/k8s/addons/azure-cloud-provider.yaml +++ b/parts/k8s/addons/azure-cloud-provider.yaml @@ -152,6 +152,7 @@ allowedTopologies: {{else}} volumeBindingMode: Immediate {{- end}} + {{- if not IsAzureStackCloud}} --- apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -165,6 +166,7 @@ parameters: reclaimPolicy: Delete allowVolumeExpansion: true volumeBindingMode: Immediate + {{- end}} {{else}} {{- if NeedsStorageAccountStorageClasses}} --- diff --git a/parts/k8s/addons/cloud-node-manager.yaml b/parts/k8s/addons/cloud-node-manager.yaml index b28ce00af0..9789c75790 100644 --- a/parts/k8s/addons/cloud-node-manager.yaml +++ b/parts/k8s/addons/cloud-node-manager.yaml @@ -88,11 +88,22 @@ spec: command: - cloud-node-manager - --node-name=$(NODE_NAME) + {{- if IsAzureStackCloud}} + - --use-instance-metadata=false + - --cloud-config=/etc/kubernetes/azure.json + - --kubeconfig=/var/lib/kubelet/kubeconfig + {{end}} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + {{- if IsAzureStackCloud}} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} resources: requests: cpu: 50m @@ -100,6 +111,29 @@ spec: limits: cpu: 2000m memory: 512Mi + {{- if IsAzureStackCloud}} + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: path-kubeconfig + mountPath: /var/lib/kubelet/kubeconfig + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: path-kubeconfig + hostPath: + path: /var/lib/kubelet/kubeconfig + type: FileOrCreate + {{end}} {{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}} --- apiVersion: apps/v1 @@ -148,11 +182,27 @@ spec: command: - /cloud-node-manager.exe - --node-name=$(NODE_NAME) + - --kubeconfig=C:\k\config + {{- if IsAzureStackCloud}} + - --use-instance-metadata=false + - --cloud-config=C:\k\azure.json + lifecycle: + postStart: + exec: + command: + - C:\k\addazsroot.bat + {{end}} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + {{- if IsAzureStackCloud}} + - name: AZURE_ENVIRONMENT_FILEPATH + value: C:\k\azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} resources: requests: cpu: 50m @@ -160,4 +210,12 @@ spec: limits: cpu: 2000m memory: 512Mi -{{end}} + volumeMounts: + - name: azure-config + mountPath: C:\k + volumes: + - name: azure-config + hostPath: + path: C:\k + type: Directory +{{end}} \ No newline at end of file diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index 31b573553b..d3690dc481 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -441,6 +441,44 @@ try Register-NodeResetScriptTask Update-DefenderPreferences + {{if IsAzureStackCloud}} + {{if UseCloudControllerManager}} + # Export the Azure Stack root cert for use in cloud node manager container setup. + $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") + if (Test-Path -Path $azsConfigFile) { + $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json + if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) { + $azsARMUri = [System.Uri]$azsJson.managementPortalURL + $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")} + if ($null -ne $azsRootCert) { + $azsRootCertFilePath = [io.path]::Combine($global:KubeDir, "azsroot.cer") + Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT + } else { + throw "$azsRootCert is null, cannot export Azure Stack root cert" + } + } else { + throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri" + } + } else { + throw "$azsConfigFile does not exist, cannot export Azure Stack root cert" + } + + # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory + $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe") + if (Test-Path -Path $certocSourcePath) { + Copy-Item -Path $certocSourcePath -Destination $global:KubeDir + } + + # Create add cert script + $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat") + if ($null -ne $azsRootCert) { + [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}") + } else { + throw "$azsRootCertFilePath is null, cannot create add cert script" + } + {{end}} + {{end}} + if (Test-Path $CacheDir) { Write-Log "Removing aks-engine bits cache directory" @@ -469,4 +507,4 @@ catch Write-Error $_ throw $_ -} +} \ No newline at end of file diff --git a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml index 87ae8a465e..65f7cc6e07 100644 --- a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml +++ b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml @@ -13,6 +13,13 @@ spec: - name: cloud-controller-manager image: {{ContainerImage "cloud-controller-manager"}} imagePullPolicy: IfNotPresent + {{- if IsAzureStackCloud}} + env: + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} command: [{{ContainerConfig "command"}}] args: [{{GetCloudControllerManagerArgs}}] resources: diff --git a/pkg/api/addons.go b/pkg/api/addons.go index 5eca288fb3..b9559535ec 100644 --- a/pkg/api/addons.go +++ b/pkg/api/addons.go @@ -581,7 +581,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) { defaultAzureFileCSIDriverAddonsConfig := KubernetesAddon{ Name: common.AzureFileCSIDriverAddonName, - Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName)), + Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName) && !cs.Properties.IsAzureStackCloud()), Containers: []KubernetesContainerSpec{ { Name: common.CSIProvisionerContainerName, @@ -1176,7 +1176,7 @@ func getCSISidecarComponent(csiDriverName, csiSidecarName string, k8sComponents // Otherwise, it returns empty string. // Azure Stack needs the '-azs' suffix so kube-proxy's manifests uses the custom hyperkube image present in the VHD func kubeProxyImageSuffix(cs ContainerService) string { - if cs.Properties.IsAzureStackCloud() { + if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") { return common.AzureStackSuffix } return "" diff --git a/pkg/api/components.go b/pkg/api/components.go index 5947c47728..2c9c2feaac 100644 --- a/pkg/api/components.go +++ b/pkg/api/components.go @@ -307,7 +307,7 @@ func getComponentDefaultContainerImage(component string, cs *ContainerService) s // componentImageSuffix returns '-azs' if target cloud is Azure Stack. Otherwise, it returns empty string. func componentImageSuffix(cs ContainerService) string { - if cs.Properties.IsAzureStackCloud() { + if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") { return common.AzureStackSuffix } return "" diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 67bcbfa0be..a429a59630 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -45,8 +45,8 @@ const ( csiSnapshotControllerImageReference string = "oss/kubernetes-csi/snapshot-controller:v2.0.0" csiAzureDiskImageReference string = "k8s/csi/azuredisk-csi:v0.7.0" csiAzureFileImageReference string = "k8s/csi/azurefile-csi:v0.6.0" - azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v0.5.1" - azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v0.5.1" + azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.1" + azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v1.1.1" dashboardImageReference string = "mcr.microsoft.com/oss/kubernetes/dashboard:v2.0.4" // deprecated dashboardMetricsScraperImageReference string = "mcr.microsoft.com/oss/kubernetes/metrics-scraper:v1.0.4" kubeFlannelImageReference string = "quay.io/coreos/flannel:v0.8.0-amd64" @@ -571,7 +571,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference, common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference, common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", - common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip", + common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", common.DashboardAddonName: dashboardImageReference, common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference, common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType), @@ -655,7 +655,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference, common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference, common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", - common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip", + common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", common.DashboardAddonName: dashboardImageReference, common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference, common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType), diff --git a/pkg/api/types.go b/pkg/api/types.go index c7596a728e..f48707512b 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1919,7 +1919,7 @@ func (p *Properties) GetCustomCloudSourcesList() string { // GetKubernetesVersion returns the cluster Kubernetes version, with the Azure Stack suffix if Azure Stack Cloud. func (p *Properties) GetKubernetesVersion() string { - if p.IsAzureStackCloud() { + if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") { return p.OrchestratorProfile.OrchestratorVersion + AzureStackSuffix } return p.OrchestratorProfile.OrchestratorVersion @@ -1930,7 +1930,7 @@ func (p *Properties) GetKubernetesHyperkubeSpec() string { var kubernetesHyperkubeSpec string k8sComponents := GetK8sComponentsByVersionMap(p.OrchestratorProfile.KubernetesConfig)[p.OrchestratorProfile.OrchestratorVersion] kubernetesHyperkubeSpec = p.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + k8sComponents["hyperkube"] - if p.IsAzureStackCloud() { + if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") { kubernetesHyperkubeSpec = kubernetesHyperkubeSpec + AzureStackSuffix } if p.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" { diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index 9c84e9f074..5839d77666 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -698,7 +698,7 @@ version = 2 hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase k8sComponents := api.GetK8sComponentsByVersionMap(cs.Properties.OrchestratorProfile.KubernetesConfig)[cs.Properties.OrchestratorProfile.OrchestratorVersion] hyperkubeImage := hyperkubeImageBase + k8sComponents[common.Hyperkube] - if cs.Properties.IsAzureStackCloud() { + if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") { hyperkubeImage = hyperkubeImage + common.AzureStackSuffix } if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" { diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 59dddf4a25..0c40257e2e 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -1848,6 +1848,7 @@ allowedTopologies: {{else}} volumeBindingMode: Immediate {{- end}} + {{- if not IsAzureStackCloud}} --- apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -1861,6 +1862,7 @@ parameters: reclaimPolicy: Delete allowVolumeExpansion: true volumeBindingMode: Immediate + {{- end}} {{else}} {{- if NeedsStorageAccountStorageClasses}} --- @@ -7231,11 +7233,22 @@ spec: command: - cloud-node-manager - --node-name=$(NODE_NAME) + {{- if IsAzureStackCloud}} + - --use-instance-metadata=false + - --cloud-config=/etc/kubernetes/azure.json + - --kubeconfig=/var/lib/kubelet/kubeconfig + {{end}} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + {{- if IsAzureStackCloud}} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} resources: requests: cpu: 50m @@ -7243,6 +7256,29 @@ spec: limits: cpu: 2000m memory: 512Mi + {{- if IsAzureStackCloud}} + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: path-kubeconfig + mountPath: /var/lib/kubelet/kubeconfig + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: path-kubeconfig + hostPath: + path: /var/lib/kubelet/kubeconfig + type: FileOrCreate + {{end}} {{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}} --- apiVersion: apps/v1 @@ -7291,11 +7327,27 @@ spec: command: - /cloud-node-manager.exe - --node-name=$(NODE_NAME) + - --kubeconfig=C:\k\config + {{- if IsAzureStackCloud}} + - --use-instance-metadata=false + - --cloud-config=C:\k\azure.json + lifecycle: + postStart: + exec: + command: + - C:\k\addazsroot.bat + {{end}} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + {{- if IsAzureStackCloud}} + - name: AZURE_ENVIRONMENT_FILEPATH + value: C:\k\azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} resources: requests: cpu: 50m @@ -7303,8 +7355,15 @@ spec: limits: cpu: 2000m memory: 512Mi -{{end}} -`) + volumeMounts: + - name: azure-config + mountPath: C:\k + volumes: + - name: azure-config + hostPath: + path: C:\k + type: Directory +{{end}}`) func k8sAddonsCloudNodeManagerYamlBytes() ([]byte, error) { return _k8sAddonsCloudNodeManagerYaml, nil @@ -17597,6 +17656,44 @@ try Register-NodeResetScriptTask Update-DefenderPreferences + {{if IsAzureStackCloud}} + {{if UseCloudControllerManager}} + # Export the Azure Stack root cert for use in cloud node manager container setup. + $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") + if (Test-Path -Path $azsConfigFile) { + $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json + if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) { + $azsARMUri = [System.Uri]$azsJson.managementPortalURL + $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")} + if ($null -ne $azsRootCert) { + $azsRootCertFilePath = [io.path]::Combine($global:KubeDir, "azsroot.cer") + Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT + } else { + throw "$azsRootCert is null, cannot export Azure Stack root cert" + } + } else { + throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri" + } + } else { + throw "$azsConfigFile does not exist, cannot export Azure Stack root cert" + } + + # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory + $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe") + if (Test-Path -Path $certocSourcePath) { + Copy-Item -Path $certocSourcePath -Destination $global:KubeDir + } + + # Create add cert script + $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat") + if ($null -ne $azsRootCert) { + [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}") + } else { + throw "$azsRootCertFilePath is null, cannot create add cert script" + } + {{end}} + {{end}} + if (Test-Path $CacheDir) { Write-Log "Removing aks-engine bits cache directory" @@ -17625,8 +17722,7 @@ catch Write-Error $_ throw $_ -} -`) +}`) func k8sKuberneteswindowssetupPs1Bytes() ([]byte, error) { return _k8sKuberneteswindowssetupPs1, nil @@ -17727,6 +17823,13 @@ spec: - name: cloud-controller-manager image: {{ContainerImage "cloud-controller-manager"}} imagePullPolicy: IfNotPresent + {{- if IsAzureStackCloud}} + env: + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + - name: AZURE_GO_SDK_LOG_LEVEL + value: INFO + {{end}} command: [{{ContainerConfig "command"}}] args: [{{GetCloudControllerManagerArgs}}] resources: diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index 58de183a04..c81f0ec1ea 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -247,7 +247,7 @@ done # Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager CLOUD_MANAGER_VERSIONS=" -0.5.1 +1.1.1 " for CLOUD_MANAGER_VERSION in ${CLOUD_MANAGER_VERSIONS}; do for COMPONENT in azure-cloud-controller-manager azure-cloud-node-manager; do