CI: [CNI] Replace the bash scripts for CNI load testing with golang t…

…est cases (#2003) CI:[CNI] Replace the bash scripts with the golang test cases
Azure · Jun 9, 2023 · 008ae45 · 008ae45
1 parent a68d638
commit 008ae45
Show file tree

Hide file tree

Showing 21 changed files with 741 additions and 334 deletions.
diff --git a/.pipelines/cni/cilium/cilium-cni-load-test.yaml b/.pipelines/cni/cilium/cilium-cni-load-test.yaml
@@ -16,7 +16,7 @@ stages:
               inlineScript: |
                 set -ex
                 make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
-                make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP} NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
+                make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision) NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
             name: "CreateAksCluster"
             displayName: "Create AKS Cluster"
   - stage: install_cilium
@@ -35,7 +35,7 @@ stages:
               inlineScript: |
                 set -ex
                 az extension add --name aks-preview
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                 ls -lah
                 pwd
                 kubectl cluster-info
@@ -72,10 +72,9 @@ stages:
               inlineScript: |
                 set -ex
                 az extension add --name aks-preview
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
-                kubectl apply -f hack/manifests/pod.yaml
-                kubectl apply -f hack/manifests/hostprocess.yaml
-                bash hack/scripts/scale_deployment.sh
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
+                cd test/integration/load
+                go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=4 -scaleup=2400
   - stage: validate_state
     dependsOn: pod_deployment
     displayName: "Validate State"
@@ -89,9 +88,9 @@ stages:
               scriptType: "bash"
               addSpnToEnvironment: true
               inlineScript: |
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                 kubectl get pods -A
-                bash hack/scripts/validate_state.sh
+                make test-validate-state
             name: "ValidateState"
             displayName: "Validate State"
             retryCountOnTaskFailure: 3
@@ -109,18 +108,22 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 echo "Scale up the pods and immediated restart the nodes"
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                 make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
+                cd test/integration/load
                 echo "Scaling the pods down to 100 per node"
-                bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -s
+                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true
+                cd ../../../
                 echo "Restarting the nodes"
                 vmss_name=$(az vmss list -g MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION) --query "[].name" -o tsv)
-                make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP} REGION=$(LOCATION) VMSS_NAME=$vmss_name
-                bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -c
+                make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision) REGION=$(LOCATION) VMSS_NAME=$vmss_name
+                cd test/integration/load
+                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000
             name: "RestartNodes"
             displayName: "Restart Nodes"
           - script: |
-              bash hack/scripts/validate_state.sh -r true
+              export RESTART_CASE=true
+              make test-validate-state
             name: "ValidateState"
             displayName: "Validate State"
             retryCountOnTaskFailure: 3
@@ -148,11 +151,11 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 set -ex
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
             name: "GetCluster"
             displayName: "Get AKS Cluster"
           - script: |
-              kubectl delete deployment container -n default
+              kubectl delete ns load-test
               cilium connectivity test
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
@@ -175,9 +178,9 @@ stages:
                 if [ "$(DELETE_RESOURCES)" ]
                 then
                   echo "Deleting Cluster and resource group"
-                  make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                  make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                   make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
-                  make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}
+                  make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision)
                   echo "Cluster and resources down"
                 else
                   echo "Deletion of resources is False"

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -131,13 +131,12 @@ steps:
 
   - script: |
       echo "validate pod IP assignment and check systemd-networkd restart"
-      kubectl apply -f hack/manifests/hostprocess.yaml
       kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
+      make test-validate-state
       echo "delete cilium connectivity test resources and re-validate state"
       kubectl delete ns cilium-test
       kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
+      make test-validate-state
     name: "validatePods"
     displayName: "Validate Pods"
   

diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -126,13 +126,12 @@ steps:
 
   - script: |
       echo "validate pod IP assignment and check systemd-networkd restart"
-      kubectl apply -f hack/manifests/hostprocess.yaml
       kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
+      make test-validate-state
       echo "delete cilium connectivity test resources and re-validate state"
       kubectl delete ns cilium-test
       kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
+      make test-validate-state
     name: "validatePods"
     displayName: "Validate Pods"
   

diff --git a/Makefile b/Makefile
@@ -709,6 +709,8 @@ workspace: ## Set up the Go workspace.
 ##@ Test 
 
 COVER_PKG ?= .
+#Restart case is used for cni load test pipeline for restarting the nodes cluster.
+RESTART_CASE ?= false
 
 # COVER_FILTER omits folders with all files tagged with one of 'unit', '!ignore_uncovered', or '!ignore_autogenerated'
 test-all: ## run all unit tests.
@@ -721,6 +723,10 @@ test-integration: ## run all integration tests.
 		CNS_VERSION=$(CNS_VERSION) \
 		go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
 
+test-validate-state:
+	cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE)
+	cd ../../..
+
 test-cyclonus: ## run the cyclonus test for npm.
 	cd test/cyclonus && bash ./test-cyclonus.sh
 	cd ..

diff --git a/hack/scripts/scale_deployment.sh b/hack/scripts/scale_deployment.sh
diff --git a/hack/scripts/validate_state.sh b/hack/scripts/validate_state.sh