Introduce automated performance testing. (#1068)

* Add version info to file, display start of performance tests. * Scale up node group before running 5000 pod test. * Create unique mng names. * Update data files for performance tests. * Add failure checking for performance tests. * Upload files to corresponding folders in s3 bucket. * Check for slow performance update. * Weekly performance test (midnight Wednesday)
aws · Jul 23, 2020 · 9dae761 · 9dae761
1 parent a1e94bf
commit 9dae761
Show file tree

Hide file tree

Showing 10 changed files with 573 additions and 18 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -82,6 +82,70 @@ jobs:
       - store_artifacts:
           path: /tmp/cni-test
 
+  performance_test:
+    docker:
+      - image: circleci/golang:1.13-stretch
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    environment:
+      <<: *env
+      RUN_CONFORMANCE: "false"
+      RUN_PERFORMANCE_TESTS: "true"
+    steps:
+      - checkout
+      - setup_remote_docker
+      - aws-cli/setup:
+          profile-name: awstester
+      - restore_cache:
+          keys:
+            - dependency-packages-store-{{ checksum "test/integration/go.mod" }}
+            - dependency-packages-store-
+      - k8s/install-kubectl:
+          # requires 1.14.9 for k8s testing, since it uses log api.
+          kubectl-version: v1.14.9
+      - run:
+          name: Run the integration tests
+          command: ./scripts/run-integration-tests.sh
+          no_output_timeout: 15m
+      - save_cache:
+          key: dependency-packages-store-{{ checksum "test/integration/go.mod" }}
+          paths:
+            - /go/pkg
+          when: always
+      - store_artifacts:
+          path: /tmp/cni-test
+
+  kops_test:
+    docker:
+      - image: circleci/golang:1.13-stretch
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    environment:
+      <<: *env
+      RUN_CONFORMANCE: "false"
+      RUN_KOPS_TEST: "true"
+    steps:
+      - checkout
+      - setup_remote_docker
+      - aws-cli/setup:
+          profile-name: awstester
+      - restore_cache:
+          keys:
+            - dependency-packages-store-{{ checksum "test/integration/go.mod" }}
+            - dependency-packages-store-
+      - k8s/install-kubectl:
+          # requires 1.14.9 for k8s testing, since it uses log api.
+          kubectl-version: v1.14.9
+      - run:
+          name: Run the integration tests
+          command: ./scripts/run-integration-tests.sh
+          no_output_timeout: 15m
+      - save_cache:
+          key: dependency-packages-store-{{ checksum "test/integration/go.mod" }}
+          paths:
+            - /go/pkg
+          when: always
+      - store_artifacts:
+          path: /tmp/cni-test
+
 workflows:
   version: 2
   check:
@@ -118,3 +182,18 @@ workflows:
                 - master
     jobs:
       - integration_test
+
+  # triggers weekly tests on master (Friday at 11 PM PST)
+  weekly-test-run:
+    triggers:
+      - schedule:
+          cron: "0 6 * * 6"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - performance_test
+      - kops_test:
+          requires:
+            - performance_test
diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go
@@ -217,13 +217,13 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 
 	ips := []*current.IPConfig{
 		{
-			Version:   "4",
-			Address:   *addr,
+			Version: "4",
+			Address: *addr,
 		},
 	}
 
 	result := &current.Result{
-		IPs:        ips,
+		IPs: ips,
 	}
 
 	return cniTypes.PrintResult(result, conf.CNIVersion)

diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
@@ -12,6 +12,15 @@ function down-test-cluster() {
 }
 
 function up-test-cluster() {
+    MNGS=""
+    if [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then
+        MNGS='{"cni-test-single-node-mng":{"name":"cni-test-single-node-mng","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":1,"asg-max-size":1,"asg-desired-capacity":1,"instance-types":["m5.16xlarge"],"volume-size":40}, "cni-test-multi-node-mng":{"name":"cni-test-multi-node-mng","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":1,"asg-max-size":100,"asg-desired-capacity":3,"instance-types":["m5.xlarge"],"volume-size":40}}'
+        RUN_CONFORMANCE=false
+        : "${PERFORMANCE_TEST_S3_BUCKET_NAME:=""}"
+    else
+        MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}'
+    fi
+
     echo -n "Configuring cluster $CLUSTER_NAME"
     AWS_K8S_TESTER_EKS_NAME=$CLUSTER_NAME \
         AWS_K8S_TESTER_EKS_LOG_COLOR=true \
@@ -26,7 +35,7 @@ function up-test-cluster() {
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE=$ROLE_CREATE \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN=$ROLE_ARN \
-        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}' \
+        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$MNGS \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS=true \
         AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \

diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh
@@ -25,6 +25,12 @@ function display_timelines() {
     echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds." 
     echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
     echo "TIMELINE: Current image integration tests took $CURRENT_IMAGE_INTEGRATION_DURATION seconds."
-    echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds."
+    if [[ "$RUN_CONFORMANCE" == true ]]; then
+        echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds."
+    fi
+    if [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then
+        echo "TIMELINE: Performance tests took $PERFORMANCE_DURATION seconds."
+    fi
     echo "TIMELINE: Down processes took $DOWN_DURATION seconds."
 }
+