From 7f83cff63ec4e8b4356ab5235208e21f64c858ce Mon Sep 17 00:00:00 2001 From: Kalya Subramanian Date: Tue, 24 Mar 2020 12:59:32 -0700 Subject: [PATCH] feat: Windows azure-cni with containerd --- .../kubernetes-hybrid.azure-containerd.json | 55 ++++++++++++++++++ ...kubernetes-hybrid.kubenet-containerd.json} | 0 parts/k8s/kuberneteswindowssetup.ps1 | 9 ++- parts/k8s/windowscontainerdfunc.ps1 | 57 ++++++++++--------- pkg/api/vlabs/validate.go | 3 - pkg/engine/templates_generated.go | 20 +++++-- 6 files changed, 107 insertions(+), 37 deletions(-) create mode 100644 examples/windows/kubernetes-hybrid.azure-containerd.json rename examples/windows/{kubernetes-hybrid.containerd.json => kubernetes-hybrid.kubenet-containerd.json} (100%) diff --git a/examples/windows/kubernetes-hybrid.azure-containerd.json b/examples/windows/kubernetes-hybrid.azure-containerd.json new file mode 100644 index 00000000000..46107da68d5 --- /dev/null +++ b/examples/windows/kubernetes-hybrid.azure-containerd.json @@ -0,0 +1,55 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.18", + "kubernetesConfig": { + "networkPlugin": "azure", + "containerRuntime": "containerd", + "windowsContainerdURL": "https://aksenginee2etestimages.blob.core.windows.net/test-content/windows-cri-containerd.zip", + "windowsSdnPluginURL": "https://aksenginee2etestimages.blob.core.windows.net/test-content/windows-cni-containerd.zip" + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "linuxpool1", + "count": 2, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "AvailabilitySet" + }, + { + "name": "windowspool2", + "count": 2, + "vmSize": "Standard_D2s_v3", + "availabilityProfile": "AvailabilitySet", + "osType": "Windows" + } + ], + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "replacepassword1234$", + "enableAutomaticUpdates": false, + "sshEnabled": true + }, + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/examples/windows/kubernetes-hybrid.containerd.json b/examples/windows/kubernetes-hybrid.kubenet-containerd.json similarity index 100% rename from examples/windows/kubernetes-hybrid.containerd.json rename to examples/windows/kubernetes-hybrid.kubenet-containerd.json diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index e10f752aa2e..e3f7486628b 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -231,7 +231,13 @@ try if ($useContainerD) { Write-Log "Installing ContainerD" $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew() - Install-Containerd -ContainerdUrl $global:ContainerdUrl + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath $containerdTimer.Stop() $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds) # TODO: disable/uninstall Docker later @@ -360,7 +366,6 @@ try } else { Update-WinCNI -CNIPath $global:CNIPath } - Get-HnsPsm1 -HNSModule $global:HNSModule } New-ExternalHnsNetwork diff --git a/parts/k8s/windowscontainerdfunc.ps1 b/parts/k8s/windowscontainerdfunc.ps1 index 2ea2c553a3f..506635c968a 100644 --- a/parts/k8s/windowscontainerdfunc.ps1 +++ b/parts/k8s/windowscontainerdfunc.ps1 @@ -1,50 +1,53 @@ # this is $global to persist across all functions since this is dot-sourced $global:ContainerdInstallLocation = "$Env:ProgramFiles\containerd" -function RegisterContainerDService -{ +function RegisterContainerDService { Assert-FileExists (Join-Path $global:ContainerdInstallLocation containerd.exe) Write-Host "Registering containerd as a service" $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue if ($null -ne $svc) { - & $cdbinary --unregister-service + & $cdbinary --unregister-service } & $cdbinary --register-service $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue if ($null -eq $svc) { - throw "containerd.exe did not get installed as a service correctly." + throw "containerd.exe did not get installed as a service correctly." } $svc | Start-Service if ($svc.Status -ne "Running") { - throw "containerd service is not running" + throw "containerd service is not running" } } -function Install-Containerd -{ - Param( - [Parameter(Mandatory = $true)][string] - $ContainerdUrl - ) - $zipfile = [Io.path]::Combine($ENV:TEMP, "containerd.zip") - DownloadFileOverHttp -Url $ContainerdUrl -DestinationPath $zipfile - Expand-Archive -path $zipfile -DestinationPath $global:ContainerdInstallLocation - del $zipfile +function Install-Containerd { + Param( + [Parameter(Mandatory = $true)][string] + $ContainerdUrl, + [Parameter(Mandatory = $true)][string] + $CNIBinDir, + [Parameter(Mandatory = $true)][string] + $CNIConfDir + ) + $zipfile = [Io.path]::Combine($ENV:TEMP, "containerd.zip") + DownloadFileOverHttp -Url $ContainerdUrl -DestinationPath $zipfile + Expand-Archive -path $zipfile -DestinationPath $global:ContainerdInstallLocation + del $zipfile - Add-SystemPathEntry $global:ContainerdInstallLocation + Add-SystemPathEntry $global:ContainerdInstallLocation - # TODO: remove if the node comes up without this code - # $configDir = [Io.Path]::Combine($ENV:ProgramData, "containerd") - # if (-Not (Test-Path $configDir)) { - # mkdir $configDir - # } + # TODO: remove if the node comes up without this code + # $configDir = [Io.Path]::Combine($ENV:ProgramData, "containerd") + # if (-Not (Test-Path $configDir)) { + # mkdir $configDir + # } - # TODO: call containerd.exe dump config, then modify instead of starting with hardcoded - $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml") - @" + # TODO: call containerd.exe dump config, then modify instead of starting with hardcoded + $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml") + + @" version = 2 root = "C:\\ProgramData\\containerd\\root" state = "C:\\ProgramData\\containerd\\state" @@ -131,8 +134,8 @@ oom_score = 0 runtime_root = "" privileged_without_host_devices = false [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "C:\\k\\cni" - conf_dir = "C:\\k\\cni\\config" + bin_dir = "$(($CNIBinDir).Replace("\","//"))" + conf_dir = "$(($CNIConfDir).Replace("\","//"))" max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] @@ -149,6 +152,6 @@ oom_score = 0 [plugins."io.containerd.service.v1.diff-service"] default = ["windows", "windows-lcow"] "@ | Out-File -Encoding ascii $configFile - RegisterContainerDService + RegisterContainerDService } \ No newline at end of file diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index b5c9f6881c5..24a41b57755 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1677,9 +1677,6 @@ func (a *Properties) validateContainerRuntime() error { // TODO: These validations should be relaxed once ContainerD and CNI plugins are more readily available if containerRuntime == Containerd && a.HasWindows() { - if a.OrchestratorProfile.KubernetesConfig.NetworkPlugin != "kubenet" { - return errors.Errorf("Windows only supports kubenet with containerd runtime. %q is not supported", a.OrchestratorProfile.KubernetesConfig.NetworkPlugin) - } if a.OrchestratorProfile.KubernetesConfig.WindowsContainerdURL == "" { return errors.Errorf("WindowsContainerdURL must be provided when using Windows with ContainerRuntime=containerd") } diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 5276eb5ae1c..d8ac69d155c 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -39821,7 +39821,13 @@ try if ($useContainerD) { Write-Log "Installing ContainerD" $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew() - Install-Containerd -ContainerdUrl $global:ContainerdUrl + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath $containerdTimer.Stop() $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds) # TODO: disable/uninstall Docker later @@ -39950,7 +39956,6 @@ try } else { Update-WinCNI -CNIPath $global:CNIPath } - Get-HnsPsm1 -HNSModule $global:HNSModule } New-ExternalHnsNetwork @@ -40967,7 +40972,11 @@ function Install-Containerd { Param( [Parameter(Mandatory = $true)][string] - $ContainerdUrl + $ContainerdUrl, + [Parameter(Mandatory=$true)][string] + $CNIBinDir, + [Parameter(Mandatory=$true)][string] + $CNIConfDir ) $zipfile = [Io.path]::Combine($ENV:TEMP, "containerd.zip") DownloadFileOverHttp -Url $ContainerdUrl -DestinationPath $zipfile @@ -40984,6 +40993,7 @@ function Install-Containerd # TODO: call containerd.exe dump config, then modify instead of starting with hardcoded $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml") + @" version = 2 root = "C:\\ProgramData\\containerd\\root" @@ -41071,8 +41081,8 @@ oom_score = 0 runtime_root = "" privileged_without_host_devices = false [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "C:\\k\\cni" - conf_dir = "C:\\k\\cni\\config" + bin_dir = "$(($CNIBinDir).Replace("\","//"))" + conf_dir = "$(($CNIConfDir).Replace("\","//"))" max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry]