From 2502fe75608ce20c5aec7bc6736a0da5e9207b98 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 4 Oct 2023 00:09:58 -0700 Subject: [PATCH 1/2] ci: increase timeout for CNI|NPM intergration test --- .pipelines/npm/npm-cni-integration-test.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.pipelines/npm/npm-cni-integration-test.yaml b/.pipelines/npm/npm-cni-integration-test.yaml index b9449101d1..749c2fecd5 100644 --- a/.pipelines/npm/npm-cni-integration-test.yaml +++ b/.pipelines/npm/npm-cni-integration-test.yaml @@ -16,6 +16,7 @@ jobs: displayName: "NPM k8s E2E" dependsOn: ${{ parameters.dependsOn }} condition: and( and( not(canceled()), not(failed()) ), ${{ or(contains(parameters.os_version, '2022'), eq(parameters.os, 'linux') ) }} , or( contains(variables.CONTROL_SCENARIO, 'npm') , contains(variables.CONTROL_SCENARIO, 'all') ) ) + timeoutInMinutes: 180 # This is for testing windows, may have to keep due to the 3m between the 14 tests -> results in 42m of wasted time pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -120,15 +121,17 @@ jobs: --ginkgo.focus="$focus" \ --ginkgo.skip="NetworkPolicyLegacy|SCTP" \ --kubeconfig=$HOME/.kube/config - - # Untaint Linux (system) nodes once testing is complete - if ${{ lower(eq(parameters.os, 'windows')) }} - then - kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule- - fi displayName: "Run Kubernetes e2e.test" continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.os, 'windows') }}: + - bash: | + # Untaint Linux (system) nodes once testing is complete + kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule- + + displayName: Untaint Linux Nodes + condition: always() + - bash: | npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt) mkdir -p $npmLogs From 020253d29c37ba29f0a43fa27e43856561a914be Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Thu, 5 Oct 2023 13:08:57 -0700 Subject: [PATCH 2/2] add: retry to generate NPM logs --- .pipelines/npm/npm-cni-integration-test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/npm/npm-cni-integration-test.yaml b/.pipelines/npm/npm-cni-integration-test.yaml index 749c2fecd5..14fa07dfa5 100644 --- a/.pipelines/npm/npm-cni-integration-test.yaml +++ b/.pipelines/npm/npm-cni-integration-test.yaml @@ -16,7 +16,7 @@ jobs: displayName: "NPM k8s E2E" dependsOn: ${{ parameters.dependsOn }} condition: and( and( not(canceled()), not(failed()) ), ${{ or(contains(parameters.os_version, '2022'), eq(parameters.os, 'linux') ) }} , or( contains(variables.CONTROL_SCENARIO, 'npm') , contains(variables.CONTROL_SCENARIO, 'all') ) ) - timeoutInMinutes: 180 # This is for testing windows, may have to keep due to the 3m between the 14 tests -> results in 42m of wasted time + timeoutInMinutes: 180 # This is for testing windows, due to the 3m between the 14 tests -> results in 42m of wasted time pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -140,9 +140,10 @@ jobs: npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'` # capture all logs for npmPod in $npmPodList; do - kubectl logs -n kube-system $npmPod > $npmLogs/$npmPod-logs.txt + kubectl logs -n kube-system $npmPod > $npmLogs/$npmPod-logs.txt done displayName: Generate NPM pod logs + retryCountOnTaskFailure: 3 condition: always() - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)