From 1132f40292331820c2c9f78b6b9d6b711b5f7fab Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 27 Dec 2023 15:11:13 -0800 Subject: [PATCH 1/2] ci: change CNS integration --- .../aks-swift/e2e-step-template.yaml | 30 ++---------------- .../azure-cni-overlay-e2e-step-template.yaml | 5 +-- .../cilium-overlay-e2e-step-template.yaml | 31 +------------------ .../cilium/cilium-e2e-step-template.yaml | 31 +------------------ .../dualstackoverlay-e2e-step-template.yaml | 5 +-- 5 files changed, 10 insertions(+), 92 deletions(-) diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 61b39e23bb..fbaf760ee5 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -52,28 +52,11 @@ steps: pwd kubectl cluster-info kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aksswifte2e" displayName: "Run AKS Swift E2E" - - script: | - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: aks-swift-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output" - condition: always() - - script: | echo "Run Service Conformance E2E" export PATH=${PATH}:/usr/local/bin/gsutil @@ -98,7 +81,7 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - + - script: | cd hack/scripts chmod +x async-delete-test.sh @@ -108,12 +91,3 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 3a6f529061..831a10a72e 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -34,7 +34,7 @@ steps: - ${{ if eq(parameters.os, 'linux') }}: - script: | echo "Start Integration Tests on Overlay Cluster" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true retryCountOnTaskFailure: 2 name: "integrationTest" displayName: "Run CNS Integration Tests on AKS Overlay" @@ -101,7 +101,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} + set -e + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} echo "Windows node are successfully added to v4 Overlay Cluster" kubectl cluster-info kubectl get node -owide diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 71a7bfc8bd..23fa60ae79 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -81,31 +81,11 @@ steps: else CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" - - script: | - echo "Status of the nodes and pods after the test" - kubectl get nodes -o wide - kubectl get pods -A -o wide - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: test-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" - condition: always() - - script: | kubectl get pods -A echo "Waiting < 2 minutes for cilium to be ready" @@ -203,12 +183,3 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index c09b215bc8..8e403ffd47 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -72,31 +72,11 @@ steps: - script: | echo "Start Azilium E2E Tests" kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZILIUM=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZILIUM=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E" - - script: | - echo "Status of the nodes and pods after the test" - kubectl get nodes -o wide - kubectl get pods -A -o wide - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: test-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" - condition: always() - - script: | kubectl get pods -A echo "Waiting < 2 minutes for cilium to be ready" @@ -147,12 +127,3 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index aef22612bc..4e36067a2c 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -36,7 +36,7 @@ steps: - script: | kubectl cluster-info kubectl get node - sudo -E env "PATH=$PATH" make test-integration CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true kubectl get po -owide -A retryCountOnTaskFailure: 3 name: "integrationTest" @@ -108,7 +108,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} + set -e + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} echo "Windows nodes have been successfully added to DualStack Overlay Cluster" kubectl cluster-info kubectl get node -owide From acf83b7d8c4028b0532946fd461fcb5d7cbf2c5a Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 27 Dec 2023 16:03:24 -0800 Subject: [PATCH 2/2] ci: output pod status before/after CNS --- .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 + .../azure-cni-overlay-e2e-step-template.yaml | 4 +++- .../cilium-overlay/cilium-overlay-e2e-step-template.yaml | 3 ++- .../singletenancy/cilium/cilium-e2e-step-template.yaml | 2 +- .../dualstackoverlay-e2e-step-template.yaml | 5 +++-- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index fbaf760ee5..ddb4c21925 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -58,6 +58,7 @@ steps: displayName: "Run AKS Swift E2E" - script: | + kubectl get po -owide -A echo "Run Service Conformance E2E" export PATH=${PATH}:/usr/local/bin/gsutil KUBECONFIG=~/.kube/config kubetest2 noop \ diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 831a10a72e..cd942cdb89 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -34,6 +34,7 @@ steps: - ${{ if eq(parameters.os, 'linux') }}: - script: | echo "Start Integration Tests on Overlay Cluster" + kubectl get po -owide -A sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true retryCountOnTaskFailure: 2 name: "integrationTest" @@ -47,6 +48,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -e + kubectl get po -owide -A clusterName=${{ parameters.clusterName }} echo "Restarting nodes" for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do @@ -102,7 +104,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} echo "Windows node are successfully added to v4 Overlay Cluster" kubectl cluster-info kubectl get node -owide diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 23fa60ae79..13e3387aed 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -81,13 +81,14 @@ steps: else CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi + kubectl get po -owide -A sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" - script: | - kubectl get pods -A + kubectl get po -owide -A echo "Waiting < 2 minutes for cilium to be ready" # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 8e403ffd47..f0b529bcb6 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -78,7 +78,7 @@ steps: displayName: "Run Azilium E2E" - script: | - kubectl get pods -A + kubectl get po -owide -A echo "Waiting < 2 minutes for cilium to be ready" # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 4e36067a2c..3a519f83ce 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -36,14 +36,15 @@ steps: - script: | kubectl cluster-info kubectl get node - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "integrationTest" displayName: "Run CNS Integration Tests on AKS DualStack Overlay" - script: | set -e + kubectl get po -owide -A cd test/integration/datapath echo "Dualstack Overlay Linux datapath IPv6 test" go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true @@ -109,7 +110,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} echo "Windows nodes have been successfully added to DualStack Overlay Cluster" kubectl cluster-info kubectl get node -owide