Restructure and add cleanup utilities

tests/testnet-performance/colo-cleanup.yml

@@ -0,0 +1,20 @@
+- command: "tests/testnet-performance/utils.sh logs"
+  label: "Collect logs"
+  agents:
+    - "queue=colo-deploy"
+
+- wait: ~
+  continue_on_failure: true
+
+- command: "tests/testnet-performance/utils.sh stop"
+  label: "Stop Network software"
+  agents:
+    - "queue=colo-deploy"
+
+- wait: ~
+  continue_on_failure: true
+
+- command: "tests/testnet-performance/utils.sh delete"
+  label: "Delete Testnet"
+  agents:
+    - "queue=colo-deploy"

tests/testnet-performance/colo-perf.yml

@@ -1,35 +1,35 @@
+CLOUD_PROVIDER: "colo"
+TESTNET_TAG: "colo-edge-perf-gpu-enabled"
+CHANNEL: edge
+RAMP_UP_TIME: 60
+TEST_DURATION: 300
+NUMBER_OF_VALIDATOR_NODES: 2
+NUMBER_OF_CLIENT_NODES: 1
+CLIENT_OPTIONS: "bench-tps=1=--tx_count 80000 --thread-batch-sleep-ms 1000"
+ADDITIONAL_FLAGS: ""
+
 steps:
   - command: "tests/testnet-performance/testnet-automation.sh"
     label: "COLO performance testnet on EDGE channel GPU ENABLED"
 #    branches: 'master'
-    env:
-      CLOUD_PROVIDER: "colo"
-      TESTNET_TAG: "colo-edge-perf-gpu-enabled"
-      CHANNEL: edge
-      RAMP_UP_TIME: 60
-      TEST_DURATION: 300
-      NUMBER_OF_VALIDATOR_NODES: 2
-      NUMBER_OF_CLIENT_NODES: 1
-      CLIENT_OPTIONS: "bench-tps=1=--tx_count 80000 --thread-batch-sleep-ms 1000"
-      ADDITIONAL_FLAGS: ""
     agents:
       - "queue=colo-deploy"
 
-  - wait: ~
-    continue_on_failure: true
-
-  - command: "tests/testnet-performance/testnet-automation.sh"
-    label: "COLO performance testnet on BETA channel GPU ENABLED"
-#    branches: 'v0.19'
-    env:
-      CLOUD_PROVIDER: "colo"
-      TESTNET_TAG: "colo-beta-perf-gpu-enabled"
-      CHANNEL: beta
-      RAMP_UP_TIME: 60
-      TEST_DURATION: 300
-      NUMBER_OF_VALIDATOR_NODES: 2
-      NUMBER_OF_CLIENT_NODES: 1
-      CLIENT_OPTIONS: "bench-tps=1=--tx_count 80000 --thread-batch-sleep-ms 1000"
-      ADDITIONAL_FLAGS: ""
-    agents:
-      - "queue=colo-deploy"
+#  - wait: ~
+#    continue_on_failure: true
+#
+#  - command: "tests/testnet-performance/testnet-automation.sh"
+#    label: "COLO performance testnet on BETA channel GPU ENABLED"
+##    branches: 'v0.19'
+#    env:
+#      CLOUD_PROVIDER: "colo"
+#      TESTNET_TAG: "colo-beta-perf-gpu-enabled"
+#      CHANNEL: beta
+#      RAMP_UP_TIME: 60
+#      TEST_DURATION: 300
+#      NUMBER_OF_VALIDATOR_NODES: 2
+#      NUMBER_OF_CLIENT_NODES: 1
+#      CLIENT_OPTIONS: "bench-tps=1=--tx_count 80000 --thread-batch-sleep-ms 1000"
+#      ADDITIONAL_FLAGS: ""
+#    agents:
+#      - "queue=colo-deploy"

tests/testnet-performance/testnet-automation.sh

@@ -17,34 +17,6 @@ set -e
 [[ -n $CHANNEL ]] || CHANNEL=beta
 [[ -n $ADDITIONAL_FLAGS ]] || ADDITIONAL_FLAGS=""
 
-function cleanup_testnet {
-  echo --- collect logs from remote nodes
-  rm -rf net/log
-  net/net.sh logs
-  for logfile in $(ls -A net/log) ; do
-    new_log=net/log/"$TESTNET_TAG"_"$nodeCount"-nodes_"$(basename "$logfile")"
-    cp "$logfile" "$new_log"
-    upload-ci-artifact "$new_log"
-  done
-
-  echo --- stop network software
-  net/net.sh stop -p $TESTNET_TAG
-
-  echo --- delete testnet
-  case $CLOUD_PROVIDER in
-    gce)
-      net/gce.sh delete -p $TESTNET_TAG
-      ;;
-    colo)
-      net/colo.sh delete -p $TESTNET_TAG
-      ;;
-    *)
-      echo "Error: Unsupported cloud provider: $CLOUD_PROVIDER"
-      ;;
-    esac
-}
-trap cleanup_testnet EXIT
-
 cd "$(dirname "$0")/../.."
 
 if [[ -z $USE_PREBUILT_CHANNEL_TARBALL ]]; then
@@ -72,25 +44,22 @@ fi
 TESTNET_CLOUD_ZONES=(); while read -r -d, ; do TESTNET_CLOUD_ZONES+=( "$REPLY" ); done <<< "${TESTNET_ZONES},"
 
 launchTestnet() {
-  declare nodeCount=$1
-  echo --- setup "$nodeCount" node test
-
   set -x
 
   # shellcheck disable=SC2068
-  echo --- create "$nodeCount" nodes
+  echo --- create "$NUMBER_OF_VALIDATOR_NODES" nodes
 
   case $CLOUD_PROVIDER in
     gce)
       net/gce.sh create \
         -d pd-ssd \
-        -n "$nodeCount" -c "$NUMBER_OF_CLIENT_NODES" \
+        -n "$NUMBER_OF_VALIDATOR_NODES" -c "$NUMBER_OF_CLIENT_NODES" \
         -G "$VALIDATOR_NODE_MACHINE_TYPE" \
         -p "$TESTNET_TAG" ${TESTNET_CLOUD_ZONES[@]/#/-z } "$ADDITIONAL_FLAGS"
       ;;
     colo)
       net/colo.sh create \
-        -n "$nodeCount" -c "$NUMBER_OF_CLIENT_NODES" \
+        -n "$NUMBER_OF_VALIDATOR_NODES" -c "$NUMBER_OF_CLIENT_NODES" -g \
         -p "$TESTNET_TAG" "$ADDITIONAL_FLAGS"
       ;;
     *)
@@ -102,7 +71,7 @@ launchTestnet() {
   net/init-metrics.sh -e
 
 # TODO: Calling net.sh restart instead of start until https://github.com/solana-labs/solana/issues/6216 is fixed
-  echo --- start "$nodeCount" node test
+  echo --- start "$NUMBER_OF_VALIDATOR_NODES" node test
   if [[ -n $USE_PREBUILT_CHANNEL_TARBALL ]]; then
     net/net.sh restart -t "$CHANNEL" "$maybeClientOptions" "$CLIENT_OPTIONS"
   else
@@ -145,7 +114,7 @@ launchTestnet() {
       FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
       WHERE time > now() - '"$TEST_DURATION"'s'
 
-  RESULTS_FILE="$TESTNET_TAG"_SUMMARY_STATS_"$nodeCount".log
+  RESULTS_FILE="$TESTNET_TAG"_SUMMARY_STATS_"$NUMBER_OF_VALIDATOR_NODES".log
   curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
     --data-urlencode "db=${TESTNET_TAG}" \
     --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation" |
@@ -154,11 +123,4 @@ launchTestnet() {
   upload-ci-artifact "$RESULTS_FILE"
 }
 
-# This is needed, because buildkite doesn't let us define an array of numbers.
-# The array is defined as a space separated string of numbers
-# shellcheck disable=SC2206
-nodes_count_array=($NUMBER_OF_VALIDATOR_NODES)
-
-for n in "${nodes_count_array[@]}"; do
-  launchTestnet "$n"
-done
+launchTestnet

tests/testnet-performance/utils.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")/../.."
+command=$1
+
+case $command in
+stop)
+  echo --- stop network software
+  net/net.sh stop -p $TESTNET_TAG
+  ;;
+delete)
+  echo --- delete testnet
+  case $CLOUD_PROVIDER in
+    gce)
+      net/gce.sh delete -p $TESTNET_TAG
+      ;;
+    colo)
+      net/colo.sh delete -p $TESTNET_TAG
+      ;;
+    *)
+      echo "Error: Unsupported cloud provider: $CLOUD_PROVIDER"
+      ;;
+    esac
+    ;;
+logs)
+  echo --- collect logs from remote nodes
+  rm -rf net/log
+  net/net.sh logs
+  for logfile in $(ls -A net/log) ; do
+    new_log=net/log/"$TESTNET_TAG"_"$NUMBER_OF_VALIDATOR_NODES"-nodes_"$(basename "$logfile")"
+    cp net/log/"$logfile" "$new_log"
+    upload-ci-artifact "$new_log"
+  done
+  ;;
+*)
+  echo "Internal error: Unknown command: $command"
+  exit 1
+esac