diff --git a/debug/windows/npm/README.md b/debug/windows/npm/README.md index 2286a6be9b..5b81d04266 100644 --- a/debug/windows/npm/README.md +++ b/debug/windows/npm/README.md @@ -3,5 +3,11 @@ This script will collect Windows NPM logs and the HNS and VFP state of the cluster and write them to a new local folder. ## How to collect logs +In a PowerShell terminal, navigate to the `azure-container-networking/debug/windows/npm folder`. Make sure your kubectl is configured to point to the cluster you want to collect logs from (`az aks get-credentials -g -n `) +### Windows +Run `.\win-debug.ps1`. The script will create a new folder called logs_DATE containing the results. -In a PowerShell terminal, navigate to the `azure-container-networking/debug/windows/npm folder`. Make sure your kubectl is configured to point to the cluster you want to collect logs from (`az aks get-credentials -g -n `) and run `.\win-debug.ps1`. The script will create a new folder called logs_DATE containing the results. \ No newline at end of file +### Linux +Run `.\win-debug.sh`. The script will create a new folder called logs_DATE containing the results. + +Note: You may not be able to unzip logs.zip in Linux since it was compressed in Windows. diff --git a/debug/windows/npm/pod_exec.ps1 b/debug/windows/npm/pod_exec.ps1 index 9aa114d824..9268bd5df3 100644 --- a/debug/windows/npm/pod_exec.ps1 +++ b/debug/windows/npm/pod_exec.ps1 @@ -1,5 +1,8 @@ param([string]$podIps) -$filepath = "logs" +$filepath = "npm-exec-logs" + +Write-Output "attempting to delete previous results if they exist" +Remove-Item -path $filepath -recurse $podIp = @() foreach ($r in ($podIps -split " ")) { @@ -21,6 +24,6 @@ foreach ($r in ($podIps -split " ")) { } } - Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' -Force + Compress-Archive -Path "$filepath" -DestinationPath "$filepath.zip" -Force exit diff --git a/debug/windows/npm/win-debug.ps1 b/debug/windows/npm/win-debug.ps1 index 68f23f1043..ac289e0d42 100644 --- a/debug/windows/npm/win-debug.ps1 +++ b/debug/windows/npm/win-debug.ps1 @@ -1,5 +1,10 @@ $filepath = "logs_$((Get-Date).ToString('MM-dd-yyyy'))" -kubectl get pod -A -o wide >> ( New-Item -Path ./$filepath/allpods.out -Force ) +Write-Output "gathering logs and writing to $filepath/" + +kubectl get pod -A -o wide --show-labels > ( New-Item -Path ./$filepath/allpods.out -Force ) +kubectl get netpol -A -o yaml > ( New-Item -Path ./$filepath/all-netpol-yamls.out -Force ) +kubectl describe netpol -A > ( New-Item -Path ./$filepath/all-netpol-descriptions.out -Force ) + $npmpod = kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | Select-String "npm-win" $rows = @() foreach ($row in (-split $npmpod)) { @@ -10,9 +15,10 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) { $npm = $rows[$i] $node = $rows[$i + 1] - Write-Output "Gathering logs for node $node" + Write-Output "Gathering logs. npm pod: $npm. node: $node" + kubectl logs -n kube-system $npm > $filepath/logs_$npm.out - $ip = kubectl get pod -n kube-system -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node" + $ip = kubectl get pod -A -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node" $ip = (-split $ip) [string] $ips = "" for ($j = 0; $j -lt $ip.Length; $j += 2) { @@ -21,10 +27,17 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) { else{ $ips += $ip[$j] } - } - - kubectl logs -n kube-system $npm >> $filepath/logs_$npm.out + } + echo "node $node has IPs: $ips" + + Write-Output "copying ps1 file into $npm" kubectl cp ./pod_exec.ps1 kube-system/"$npm":execw.ps1 + + Write-Output echo "executing ps1 file on $npm" kubectl exec -it -n kube-system $npm -- powershell.exe -Command .\execw.ps1 "'$ips'" - kubectl cp kube-system/"$npm":logs.zip ./$filepath/logs_$node.zip + + Write-Output "copying logs.zip from $npm" + kubectl cp kube-system/"$npm":npm-exec-logs.zip ./$filepath/npm-exec-logs_$node.zip } + +Write-Output "finished capturing all logs. written to $filepath/" diff --git a/debug/windows/npm/win-debug.sh b/debug/windows/npm/win-debug.sh new file mode 100755 index 0000000000..28f13ca20f --- /dev/null +++ b/debug/windows/npm/win-debug.sh @@ -0,0 +1,54 @@ +# NOTE: you may not be able to unzip logs.zip in Linux since it was compressed in Windows +set -e +dateString=`date -I` # like 2022-09-24 +filepath=logs_$dateString +mkdir $filepath + +echo "gathering logs and writing to $filepath/" + +kubectl get pod -A -o wide --show-labels > $filepath/allpods.out +kubectl get netpol -A -o yaml > $filepath/all-netpol-yamls.out +kubectl describe netpol -A > $filepath/all-netpol-descriptions.out + +npmPods=() +nodes=() +for npmPodOrNode in `kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | grep "npm-win"`; do + # for loop will go over each item (npm pod, then its node, then the next npm pod, then its node, ...) + set +e + echo $npmPodOrNode | grep -q azure-npm-win- + if [ $? -eq 0 ]; then + npmPods+=($npmPodOrNode) + else + nodes+=($npmPodOrNode) + fi +done +set -e + +echo "npm pods: ${npmPods[@]}" +echo "nodes of npm pods: ${nodes[@]}" + +for i in $(seq 1 ${#npmPods[*]}); do + j=$((i-1)) + npmPod=${npmPods[$j]} + node=${nodes[$j]} + + echo "gathering logs. npm pod: $npmPod. node: $node" + kubectl logs -n kube-system $npmPod > $filepath/logs_$npmPod.out + + ips=() + for ip in `kubectl get pod -A -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | grep $node | grep -oP "\d+\.\d+\.\d+\.\d+"`; do + ips+=($ip) + done + echo "node $node has IPs: ${ips[@]}" + + echo "copying ps1 file into $npmPod" + kubectl cp ./pod_exec.ps1 kube-system/"$npmPod":execw.ps1 + + echo "executing ps1 file on $npmPod" + kubectl exec -it -n kube-system $npmPod -- powershell.exe -Command .\\execw.ps1 "'${ips[@]}'" + + echo "copying logs.zip from $npmPod. NOTE: this will be a windows-based compressed archive (probably need windows to expand it)" + kubectl cp kube-system/"$npmPod":npm-exec-logs.zip $filepath/npm-exec-logs_$node.zip +done + +echo "finished gathering all logs. written to $filepath/"