From 5be76a9f822990b63658990a4e81f29046525746 Mon Sep 17 00:00:00 2001 From: marosset Date: Mon, 11 Jan 2021 23:35:16 -0800 Subject: [PATCH 1/3] fix: fix network cleanup code on windows for contianerd nodes --- staging/provisioning/windows/cleanupnetwork.ps1 | 9 +++++++++ staging/provisioning/windows/kubeletstart.ps1 | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/staging/provisioning/windows/cleanupnetwork.ps1 b/staging/provisioning/windows/cleanupnetwork.ps1 index b53e82fbf8..d9c646935f 100644 --- a/staging/provisioning/windows/cleanupnetwork.ps1 +++ b/staging/provisioning/windows/cleanupnetwork.ps1 @@ -3,6 +3,7 @@ $Global:ClusterConfiguration = ConvertFrom-Json ((Get-Content "c:\k\kubeclusterc $global:NetworkMode = "L2Bridge" $global:ContainerRuntime = $Global:ClusterConfiguration.Cri.Name $global:NetworkPlugin = $Global:ClusterConfiguration.Cni.Name +$global:HNSModule = "c:\k\hns.psm1" ipmo $global:HNSModule @@ -49,6 +50,8 @@ if ($global:NetworkPlugin -eq "azure") { taskkill /IM azure-vnet.exe /f taskkill /IM azure-vnet-ipam.exe /f + # azure-cni logs currently end up in c:\windows\system32 when machines are configured with containerd. + # https://github.com/containerd/containerd/issues/4928 $filesToRemove = @( "c:\k\azure-vnet.json", "c:\k\azure-vnet.json.lock", @@ -56,6 +59,12 @@ if ($global:NetworkPlugin -eq "azure") { "c:\k\azure-vnet-ipam.json.lock" "c:\k\azure-vnet-ipamv6.json", "c:\k\azure-vnet-ipamv6.json.lock" + "c:\windows\system32\azure-vnet.json", + "c:\windows\system32\azure-vnet.json.lock", + "c:\windows\system32\azure-vnet-ipam.json", + "c:\windows\system32\azure-vnet-ipam.json.lock" + "c:\windows\system32\azure-vnet-ipamv6.json", + "c:\windows\system32\azure-vnet-ipamv6.json.lock" ) foreach ($file in $filesToRemove) { diff --git a/staging/provisioning/windows/kubeletstart.ps1 b/staging/provisioning/windows/kubeletstart.ps1 index 8d401d1b0b..d8eac545f5 100644 --- a/staging/provisioning/windows/kubeletstart.ps1 +++ b/staging/provisioning/windows/kubeletstart.ps1 @@ -227,7 +227,7 @@ if ($global:NetworkPlugin -eq "kubenet") { $process | Stop-Process | Out-Null } - ./cleanupnetwork.ps1 + & "c:\k\cleanupnetwork.ps1" Write-Host "Creating a new hns Network" $hnsNetwork = New-HNSNetwork -Type $global:NetworkMode -AddressPrefix $podCIDR -Gateway $masterSubnetGW -Name $global:NetworkMode.ToLower() -Verbose From 291b31a79a8bfd12ddfca85f63d81e3ba959d52d Mon Sep 17 00:00:00 2001 From: marosset Date: Tue, 12 Jan 2021 09:15:39 -0800 Subject: [PATCH 2/3] updating more calls to cleanupnetwork.ps1 with full path --- staging/provisioning/windows/kubeletstart.ps1 | 4 ++-- staging/provisioning/windows/windowsnodereset.ps1 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/staging/provisioning/windows/kubeletstart.ps1 b/staging/provisioning/windows/kubeletstart.ps1 index d8eac545f5..a506403845 100644 --- a/staging/provisioning/windows/kubeletstart.ps1 +++ b/staging/provisioning/windows/kubeletstart.ps1 @@ -191,8 +191,8 @@ if ($global:NetworkPlugin -eq "azure") { # Find if network created by CNI exists, if yes, remove it # This is required to keep the network non-persistent behavior # Going forward, this would be done by HNS automatically during restart of the node - ./cleanupnetwork.ps1 - + & "c:\k\cleanupnetwork.ps1" + # Restart Kubeproxy, which would wait, until the network is created # This was fixed in 1.15, workaround still needed for 1.14 https://github.com/kubernetes/kubernetes/pull/78612 Restart-Service Kubeproxy diff --git a/staging/provisioning/windows/windowsnodereset.ps1 b/staging/provisioning/windows/windowsnodereset.ps1 index 93b29a9c95..6df9d17289 100644 --- a/staging/provisioning/windows/windowsnodereset.ps1 +++ b/staging/provisioning/windows/windowsnodereset.ps1 @@ -49,7 +49,7 @@ if ($global:EnableHostsConfigAgent) { # Perform cleanup # -./cleanupnetwork.ps1 +& "c:\k\cleanupnetwork.ps1" # # Create required networks From 62484dcdffd5d871818176d999f8c1025c0514f4 Mon Sep 17 00:00:00 2001 From: marosset Date: Tue, 12 Jan 2021 14:16:35 -0800 Subject: [PATCH 3/3] update collect-windows-logs.ps1 to grab azure-cni logs from system32 --- scripts/collect-windows-logs.ps1 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/collect-windows-logs.ps1 b/scripts/collect-windows-logs.ps1 index 007818d556..6fcefa0cd8 100644 --- a/scripts/collect-windows-logs.ps1 +++ b/scripts/collect-windows-logs.ps1 @@ -24,6 +24,20 @@ $lockedFiles | Foreach-Object { } } +# azure-cni logs currently end up in system32 when called by containerd so check there for logs too +$lockedTemp = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) +New-Item -Type Directory $lockedTemp +$lockedFiles | Foreach-Object { + Write-Host "Copying $_ to temp" + $src = "c:\windows\system32\$_" + if (Test-Path $src) { + $tempfile = Copy-Item $src $lockedTemp -Passthru -ErrorAction Ignore + if ($tempFile) { + $paths += $tempFile + } + } +} + # Containerd log is outside the c:\k folder $containerd = "C:\ProgramData\containerd\root\panic.log" if (Test-Path $containerd) {