Merge pull request #437 from natarajaya/GPII-4003

[GPII-4003]: Fix :test task to properly destroy locust resources

.gitlab-ci.yml

@@ -230,10 +230,6 @@ gcp-stg:
     - cd gcp/live/stg
     - rake display_cluster_state
     - rake display_universal_image_info || true
-    # Destroy Locust module (used for smoke tests) and its TF state
-    - rake destroy_module[locust/swarm] || true
-    - rake destroy_module[locust/istio] || true
-    - rake destroy_tfstate[locust] || true
     # Remove all SA keys except current one to prevent hitting 10 keys per SA limit (GPII-3299)
     - rake destroy_sa_keys || true
   only:
@@ -303,10 +299,6 @@ gcp-prd:
     - cd gcp/live/prd
     - rake display_cluster_state
     - rake display_universal_image_info || true
-    # Destroy Locust module (used for smoke tests) and its TF state
-    - rake destroy_module[locust/swarm] RAKE_REALLY_DESTROY_IN_PRD=true || true
-    - rake destroy_module[locust/istio] RAKE_REALLY_DESTROY_IN_PRD=true || true
-    - rake destroy_tfstate[locust] RAKE_REALLY_DESTROY_IN_PRD=true || true
     # Remove all SA keys except current one to prevent hitting 10 keys per SA limit (GPII-3299)
     - rake destroy_sa_keys RAKE_REALLY_DESTROY_IN_PRD=true || true
   only:

gcp/modules/locust/swarm.tf

@@ -103,7 +103,7 @@ resource "null_resource" "locust_swarm_session" {
         fi
         RETRY_COUNT=$((RETRY_COUNT+1))
       done
-      [ "$EXIT_STATUS" != "0" ] && echo "Failed to post resutls to Stackdriver, retry limit reached, giving up."
+      [ "$EXIT_STATUS" != "0" ] && echo "Failed to post results to Stackdriver, retry limit reached, giving up."
 
       if [ "$total_rps" -lt "${var.locust_desired_total_rps}" ]; then
         echo

shared/rakefiles/test.rake

@@ -1,21 +1,13 @@
 task :test do
-  sh "#{@exekube_cmd} rake xk['down live/#{@env}/locust',skip_secret_mgmt] || true"
+  locust_status = 0
+  sh "#{@exekube_cmd} rake xk['up live/#{@env}/locust',skip_secret_mgmt]" do |ok, res|
+    locust_status = res.exitstatus
+  end
+  # We want to clean up after Locust even if test failed
+  sh "#{@exekube_cmd} rake xk['down live/#{@env}/locust',skip_secret_mgmt]"
   Rake::Task[:destroy_tfstate].invoke('locust')
-
-  sh "#{@exekube_cmd} sh -c ' \
-      RETRIES=10; \
-      RETRY_COUNT=1; \
-      while [ \"$(kubectl get pods -n locust -o json 2> /dev/null | jq -r .items[] | grep -c .)\" != \"0\" ]; do \
-        if [ \"$RETRY_COUNT\" -gt \"$RETRIES\" ]; then \
-          echo \"Retry limit reached, giving up!\"; \
-          exit 1; \
-        fi; \
-        echo \"[Try $RETRY_COUNT of $RETRIES] Waiting for K8s to terminate Locust pods...\"; \
-        RETRY_COUNT=$(($RETRY_COUNT+1)); \
-        sleep 10; \
-      done'"
-
-  sh "#{@exekube_cmd} rake xk['up live/#{@env}/locust',skip_secret_mgmt]"
+  # Exit only if something went wrong to fail the pipeline
+  exit locust_status unless locust_status == 0
 end
 
 desc "[TEST] Run Locust swarm against Preferences service in current cluster"