From 10834ff3a05de30a89c0e9f0e412ef42de1398f0 Mon Sep 17 00:00:00 2001 From: Cristina Kovacs Date: Mon, 19 Sep 2022 14:48:41 -0500 Subject: [PATCH 1/4] add windows debug scripts --- debug/windows/npm/pod_exec.ps1 | 25 +++++++++++++++++++++++++ debug/windows/npm/win-debug.ps1 | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 debug/windows/npm/pod_exec.ps1 create mode 100644 debug/windows/npm/win-debug.ps1 diff --git a/debug/windows/npm/pod_exec.ps1 b/debug/windows/npm/pod_exec.ps1 new file mode 100644 index 0000000000..db099af9fa --- /dev/null +++ b/debug/windows/npm/pod_exec.ps1 @@ -0,0 +1,25 @@ +param([string]$podIps) +$filepath = "logs" + +$podIp = @() +foreach ($r in ($podIps -split " ")) { + $podIp += $r +} + + (Get-HnsNetwork | ? Name -Like azure).Policies >>( New-Item -Path ./$filepath/hns_state.out -Force ) + Get-HnsEndpoint | ConvertTo-Json >> $filepath/hns_state.out + + + foreach ($row in $podIp) { + [string]$endpoint = hnsdiag list endpoints | select-string -context 2, 0 "$row" + if($endpoint -ne $null){ + $endpointID = $endpoint.Substring($endpoint.IndexOf(":")+2,37).Trim() + hnsdiag list endpoints | select-string -context 2, 0 "$row" >> $filepath/vfp_state_$row.out + vfpctrl /port $endpointID /list-tag >> $filepath/vfp_state_$row.out + vfpctrl /port $endpointID /layer ACL_ENDPOINT_LAYER /list-rule >> $filepath/vfp_state_$row.out + } + } + + Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' + +exit \ No newline at end of file diff --git a/debug/windows/npm/win-debug.ps1 b/debug/windows/npm/win-debug.ps1 new file mode 100644 index 0000000000..00b0bb1591 --- /dev/null +++ b/debug/windows/npm/win-debug.ps1 @@ -0,0 +1,28 @@ +$filepath = "logs_$((Get-Date).ToString('MM-dd-yyyy'))" +kubectl get pod -A >> ( New-Item -Path ./$filepath/allpods.out -Force ) +$npmpod = kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | Select-String "npm-win" +$rows = @() +foreach ($row in (-split $npmpod)) { + $rows += $row +} + +for ($i = 0; $i -lt $rows.Length; $i += 2) { + $npm = $rows[$i] + $node = $rows[$i + 1] + + $ip = kubectl get pod -n kube-system -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node" + $ip = (-split $ip) + [string] $ips = "" + for ($j = 0; $j -lt $ip.Length; $j += 2) { + if($j -ne $ip.Length-2){ + $ips += $ip[$j] + " "} + else{ + $ips += $ip[$j] + } + } + + kubectl logs -n kube-system $npm >> $filepath/logs_$npm.out + kubectl cp ./pod_exec.ps1 kube-system/"$npm":execw.ps1 + kubectl exec -it -n kube-system $npm -- powershell.exe -Command .\execw.ps1 "'$ips'" + kubectl cp kube-system/"$npm":logs.zip ./$filepath/logs_$node.zip +} \ No newline at end of file From 01ab1412d03555c6a45d20257215ddb1226b89ac Mon Sep 17 00:00:00 2001 From: Cristina Kovacs Date: Mon, 19 Sep 2022 14:59:10 -0500 Subject: [PATCH 2/4] added newline and kubectl update --- debug/windows/npm/pod_exec.ps1 | 2 +- debug/windows/npm/win-debug.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/windows/npm/pod_exec.ps1 b/debug/windows/npm/pod_exec.ps1 index db099af9fa..50ae4daf07 100644 --- a/debug/windows/npm/pod_exec.ps1 +++ b/debug/windows/npm/pod_exec.ps1 @@ -22,4 +22,4 @@ foreach ($r in ($podIps -split " ")) { Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' -exit \ No newline at end of file +exit diff --git a/debug/windows/npm/win-debug.ps1 b/debug/windows/npm/win-debug.ps1 index 00b0bb1591..a628ebdf6e 100644 --- a/debug/windows/npm/win-debug.ps1 +++ b/debug/windows/npm/win-debug.ps1 @@ -1,5 +1,5 @@ $filepath = "logs_$((Get-Date).ToString('MM-dd-yyyy'))" -kubectl get pod -A >> ( New-Item -Path ./$filepath/allpods.out -Force ) +kubectl get pod -A -o wide >> ( New-Item -Path ./$filepath/allpods.out -Force ) $npmpod = kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | Select-String "npm-win" $rows = @() foreach ($row in (-split $npmpod)) { @@ -25,4 +25,4 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) { kubectl cp ./pod_exec.ps1 kube-system/"$npm":execw.ps1 kubectl exec -it -n kube-system $npm -- powershell.exe -Command .\execw.ps1 "'$ips'" kubectl cp kube-system/"$npm":logs.zip ./$filepath/logs_$node.zip -} \ No newline at end of file +} From b6061ea91fa95c18aa8ae521a8b7390d72b24875 Mon Sep 17 00:00:00 2001 From: Cristina Kovacs Date: Wed, 21 Sep 2022 16:13:27 -0500 Subject: [PATCH 3/4] added readme and updated archive call --- debug/windows/npm/README.md | 7 +++++++ debug/windows/npm/pod_exec.ps1 | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 debug/windows/npm/README.md diff --git a/debug/windows/npm/README.md b/debug/windows/npm/README.md new file mode 100644 index 0000000000..2286a6be9b --- /dev/null +++ b/debug/windows/npm/README.md @@ -0,0 +1,7 @@ +# Introduction + +This script will collect Windows NPM logs and the HNS and VFP state of the cluster and write them to a new local folder. + +## How to collect logs + +In a PowerShell terminal, navigate to the `azure-container-networking/debug/windows/npm folder`. Make sure your kubectl is configured to point to the cluster you want to collect logs from (`az aks get-credentials -g -n `) and run `.\win-debug.ps1`. The script will create a new folder called logs_DATE containing the results. \ No newline at end of file diff --git a/debug/windows/npm/pod_exec.ps1 b/debug/windows/npm/pod_exec.ps1 index 50ae4daf07..7e6c7b015a 100644 --- a/debug/windows/npm/pod_exec.ps1 +++ b/debug/windows/npm/pod_exec.ps1 @@ -20,6 +20,6 @@ foreach ($r in ($podIps -split " ")) { } } - Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' + Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' -Force exit From 2ec8b0d76c20d9833feb351b1ae542e1973bfee2 Mon Sep 17 00:00:00 2001 From: Cristina Kovacs Date: Wed, 21 Sep 2022 16:36:49 -0500 Subject: [PATCH 4/4] Adding terminal output --- debug/windows/npm/pod_exec.ps1 | 1 + debug/windows/npm/win-debug.ps1 | 2 ++ 2 files changed, 3 insertions(+) diff --git a/debug/windows/npm/pod_exec.ps1 b/debug/windows/npm/pod_exec.ps1 index 7e6c7b015a..9aa114d824 100644 --- a/debug/windows/npm/pod_exec.ps1 +++ b/debug/windows/npm/pod_exec.ps1 @@ -11,6 +11,7 @@ foreach ($r in ($podIps -split " ")) { foreach ($row in $podIp) { + Write-Output "Gathering logs for IP $row" [string]$endpoint = hnsdiag list endpoints | select-string -context 2, 0 "$row" if($endpoint -ne $null){ $endpointID = $endpoint.Substring($endpoint.IndexOf(":")+2,37).Trim() diff --git a/debug/windows/npm/win-debug.ps1 b/debug/windows/npm/win-debug.ps1 index a628ebdf6e..68f23f1043 100644 --- a/debug/windows/npm/win-debug.ps1 +++ b/debug/windows/npm/win-debug.ps1 @@ -10,6 +10,8 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) { $npm = $rows[$i] $node = $rows[$i + 1] + Write-Output "Gathering logs for node $node" + $ip = kubectl get pod -n kube-system -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node" $ip = (-split $ip) [string] $ips = ""