test: expand KWOK scripts to test deletions and more

test/scale/README.md

@@ -11,5 +11,36 @@ This saves us from:
 ## Usage
 1. Create AKS cluster with `--uptime-sla` and create any nodepools.
 2. If making KWOK Pods, run `run-kwok.sh` in the background.
-3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods.
+3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods. Can also delete/re-add objects to cause churn.
 4. Test connectivity with `connectivity/test-connectivity.sh`.
+
+### Example Runs
+```
+./test-scale.sh --max-kwok-pods-per-node=50 \
+    --num-kwok-deployments=10 \
+    --num-kwok-replicas=1 \
+    --max-real-pods-per-node=30 \
+    --num-real-deployments=5 \
+    --num-real-replicas=2 \
+    --num-network-policies=1 \
+    --num-unapplied-network-policies=10 \
+    --num-unique-labels-per-pod=2 \
+    --num-unique-labels-per-deployment=2 \
+    --num-shared-labels-per-pod=10 \
+    --delete-kwok-pods=10 \
+    --delete-real-pods=5 \
+    --delete-pods-interval=120 \
+    --delete-pods-times=2 \
+    --delete-labels \
+    --delete-labels-interval=30 \
+    --delete-labels-times=2 \
+    --delete-netpols \
+    --delete-netpols-interval=0 \
+    --delete-netpols-times=1
+```
+
+```
+./test-connectivity --num-scale-pods-to-verify=10 \
+    --max-wait-for-initial-connectivity=600 \
+    --max-wait-after-adding-netpol=120
+```

test/scale/connectivity/test-connectivity.sh

@@ -10,7 +10,7 @@ NETPOL_SLEEP=5
 
 printHelp() {
     cat <<EOF
-./test-connectivity.sh --num-scale-pods-to-verify=<int> --max-wait-for-initial-connectivity=<int> --max-wait-after-adding-netpol=<int> [--kubeconfig=<path>]
+./test-connectivity.sh --num-scale-pods-to-verify=all|<int> --max-wait-for-initial-connectivity=<int> --max-wait-after-adding-netpol=<int> [--kubeconfig=<path>]
 
 Verifies that scale test Pods can connect to each other, but cannot connect to a new "pinger" Pod.
 Then, adds a NetworkPolicy to allow traffic between the scale test Pods and the "pinger" Pod, and verifies connectivity.
@@ -21,7 +21,7 @@ USAGE:
 3. Run this script
 
 REQUIRED PARAMETERS:
-    --num-scale-pods-to-verify=<int>             number of scale Pods to test. Will verify that each scale Pod can connect to each other [(N-1)^2 connections] and that each Scale Pod cannot connect to a "pinger" Pod [2N connection attempts with a 3-second timeout]
+    --num-scale-pods-to-verify=all|<int>         number of scale Pods to test. Will verify that each scale Pod can connect to each other [(N-1)^2 connections] and that each Scale Pod cannot connect to a "pinger" Pod [2N connection attempts with a 3-second timeout]
     --max-wait-for-initial-connectivity=<int>    maximum time in seconds to wait for initial connectivity after Pinger Pods are running
     --max-wait-after-adding-netpol=<int>         maximum time in seconds to wait for allowed connections after adding the allow-pinger NetworkPolicy
 
@@ -72,7 +72,7 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-if [[ -z $numScalePodsToVerify || -z $maxWaitAfterAddingNetpol ]]; then
+if [[ -z $numScalePodsToVerify || -z $maxWaitForInitialConnectivity || -z $maxWaitAfterAddingNetpol ]]; then
     echo "ERROR: missing required parameter. Check --help for usage"
     exit 6
 fi
@@ -122,17 +122,20 @@ startDate=`date -u`
 echo "STARTING CONNECTIVITY TEST at $startDate"
 
 ## GET SCALE PODS
+if [[ $numScalePodsToVerify == "all" ]]; then
+    echo "setting numScalePodsToVerify=9999 since 'all' was passed in"
+    numScalePodsToVerify=9999
+fi
+
 echo "getting scale Pods..."
-scalePodNameIPs=(`kubectl $KUBECONFIG_ARG get pods -n scale-test --field-selector=status.phase==Running -o jsonpath='{range .items[*]}{@.metadata.name}{","}{@.status.podIP}{" "}{end}'`)
+scalePodNameIPs=(`kubectl $KUBECONFIG_ARG get pods -n scale-test --field-selector=status.phase==Running -l is-real="true" -o jsonpath='{range .items[*]}{@.metadata.name}{","}{@.status.podIP}{" "}{end}'`)
 scalePods=()
 scalePodIPs=()
 for nameIP in "${scalePodNameIPs[@]}"; do
     nameIP=(`echo $nameIP | tr ',' ' '`)
     name=${nameIP[0]}
     ip=${nameIP[1]}
 
-    echo $name | grep real-dep || continue
-
     echo "scale Pod: $name, IP: $ip"
 
     if [[ -z $name || -z $ip ]]; then
@@ -151,6 +154,7 @@ done
 numScalePodsFound=${#scalePods[@]}
 if [[ $numScalePodsFound == 0 ]]; then
     echo "ERROR: expected namespace scale-test to exist with real (non-kwok) Pods. Run test/scale/test-scale.sh with real Pods first."
+    kubectl get pod -n scale-test -owide
     exit 7
 elif [[ $numScalePodsFound -lt $numScalePodsToVerify ]]; then
     echo "WARNING: there are only $numScalePodsFound real scale Pods running which is less than numScalePodsToVerify=$numScalePodsToVerify. Will verify just these $numScalePodsFound Pods"
@@ -255,7 +259,7 @@ done
 
 low=0
 if [[ $prevTryDate -gt $connectivityStartDate ]]; then
-    low=$(( `date +%s` - $prevTryDate - $CONNECTIVITY_SLEEP ))
+    low=$(( $prevTryDate - $connectivityStartDate - $CONNECTIVITY_SLEEP ))
 fi
 high=$(( `date +%s` - $connectivityStartDate ))
 echo "SUCCESS: all initial connectivity tests passed. Took between $low and $high seconds to succeed"
@@ -309,7 +313,7 @@ done
 
 low=0
 if [[ $prevTryDate -gt $netpolStartDate ]]; then
-    low=$(( `date +%s` - $prevTryDate - $NETPOL_SLEEP ))
+    low=$(( $prevTryDate - $connectivityStartDate - $NETPOL_SLEEP ))
 fi
 high=$(( `date +%s` - $netpolStartDate ))
 echo "SUCCESS: all connectivity tests passed after adding allow-pinger NetworkPolicy. Took between $low and $high seconds to take effect"

test/scale/install-kwok.sh

@@ -0,0 +1,5 @@
+KWOK_REPO=kubernetes-sigs/kwok
+KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
+wget -O kwok -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_LATEST_RELEASE}/kwok-$(go env GOOS)-$(go env GOARCH)"
+chmod +x kwok
+sudo mv kwok /usr/local/bin/kwok

test/scale/run-kwok.sh

@@ -1,33 +1,21 @@
-######################################################################################
-# This script is used to schedule kwok nodes/pods and maintain kwok node heartbeats. #
-######################################################################################
-INSTALL_KWOK=false
-# KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
-KWOK_VERSION=${KWOK_LATEST_RELEASE:-"v0.1.1"}
-# kubeconfig arg doesn't seem to work for kwok. It seems to just use current context of the default kubeconfig.
-
-# specify kubeconfig file as first arg if you want
-if [[ $1 != "" ]]; then
-    file=$1
-    test -f $file || {
-        echo "ERROR: KUBECONFIG=$file does not exist"
-        exit 1
-    }
-
-    KUBECONFIG_ARG="--kubeconfig $file"
+###############################################################
+# Schedule kwok nodes/pods and maintain kwok node heartbeats. #
+###############################################################
+# can pass kubeconfig as first arg
+if [[ -z $1 ]]; then
+    kubeconfigFile=~/.kube/config
+else
+    kubeconfigFile=$1
 fi
+echo "using kubeconfig $kubeconfigFile"
 
-if [[ INSTALL_KWOK == true ]]; then
-    wget -O kwokctl -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_VERSION}/kwokctl-$(go env GOOS)-$(go env GOARCH)"
-    chmod +x kwokctl
-    sudo mv kwokctl /usr/local/bin/kwokctl
-
-    wget -O kwok -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_VERSION}/kwok-$(go env GOOS)-$(go env GOARCH)"
-    chmod +x kwok
-    sudo mv kwok /usr/local/bin/kwok
-fi
+which kwok || {
+    echo "ERROR: kwok not found. Install via ./install-kwok.sh"
+    exit 1
+}
 
-kwok $KUBECONFIG_ARG \
+set -x
+kwok --kubeconfig $kubeconfigFile \
     --cidr=155.0.0.0/16 \
     --node-ip=155.0.0.1 \
     --manage-all-nodes=false \

test/scale/templates/kwok-deployment.yaml

@@ -5,15 +5,18 @@ metadata:
   namespace: scale-test
   labels:
     app: scale-test
+    is-kwok: "true"
 spec:
   replicas: TEMP_REPLICAS
   selector:
     matchLabels:
-      app: scale-testOTHER_LABELS_6_SPACES
+      app: scale-test
+      is-kwok: "true"OTHER_LABELS_6_SPACES
   template:
     metadata:
       labels:
-        app: scale-testOTHER_LABELS_8_SPACES
+        app: scale-test
+        is-kwok: "true"OTHER_LABELS_8_SPACES
     spec:
       affinity:
         nodeAffinity:

test/scale/templates/real-deployment.yaml

@@ -5,15 +5,18 @@ metadata:
   namespace: scale-test
   labels:
     app: scale-test
+    is-real: "true"
 spec:
   replicas: TEMP_REPLICAS
   selector:
     matchLabels:
-      app: scale-testOTHER_LABELS_6_SPACES
+      app: scale-test
+      is-real: "true"OTHER_LABELS_6_SPACES
   template:
     metadata:
       labels:
-        app: scale-testOTHER_LABELS_8_SPACES
+        app: scale-test
+        is-real: "true"OTHER_LABELS_8_SPACES
     spec:
       nodeSelector:
         scale-test: "true"

test/scale/templates/unapplied-networkpolicy.yaml

@@ -0,0 +1,22 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: TEMP_NAME
+  namespace: scale-test
+spec:
+  podSelector:
+    matchLabels:
+      non-existent-key: val
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          non-existent-key: val
+  egress:
+  - to:
+    - podSelector:
+        matchLabels:
+          non-existent-key: val
+  policyTypes:
+  - Ingress
+  - Egress