From 219068f272d14dd14ceefc80fdd8d2bf58c741be Mon Sep 17 00:00:00 2001
From: Hunter Gregory <42728408+huntergregory@users.noreply.github.com>
Date: Tue, 11 Apr 2023 16:48:14 -0700
Subject: [PATCH 01/20] correct num acls

---
 test/scale/test-scale.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index ce87775e56..981d505348 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -122,7 +122,7 @@ numTotalPods=$(( $numKwokPods + $numRealPods ))
 
 ## NPM CALCULATIONS
 # unique to templates/networkpolicy.yaml
-numACLsAddedByNPM=$(( 4 * $numNetworkPolicies ))
+numACLsAddedByNPM=$(( 6 * $numNetworkPolicies ))
 # IPSet/member counts can be slight underestimates if there are more than one template-hash labels
 # 4 basic IPSets are [ns-scale-test,kubernetes.io/metadata.name:scale-test,template-hash:xxxx,app:scale-test]
 numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment ))

From bb221c2fa0b6906615893ab474b044591c0f1f58 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <42728408+huntergregory@users.noreply.github.com>
Date: Tue, 11 Apr 2023 17:41:08 -0700
Subject: [PATCH 02/20] print out pods for debugging

---
 test/scale/connectivity/test-connectivity.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/scale/connectivity/test-connectivity.sh b/test/scale/connectivity/test-connectivity.sh
index 75ab9256ff..85f72f054b 100755
--- a/test/scale/connectivity/test-connectivity.sh
+++ b/test/scale/connectivity/test-connectivity.sh
@@ -151,6 +151,7 @@ done
 numScalePodsFound=${#scalePods[@]}
 if [[ $numScalePodsFound == 0 ]]; then
     echo "ERROR: expected namespace scale-test to exist with real (non-kwok) Pods. Run test/scale/test-scale.sh with real Pods first."
+    kubectl get pod -n scale-test -owide
     exit 7
 elif [[ $numScalePodsFound -lt $numScalePodsToVerify ]]; then
     echo "WARNING: there are only $numScalePodsFound real scale Pods running which is less than numScalePodsToVerify=$numScalePodsToVerify. Will verify just these $numScalePodsFound Pods"

From d941a7e0f6e911ba094e030583f2f5660796c051 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Tue, 11 Apr 2023 18:24:25 -0700
Subject: [PATCH 03/20] fix latency value printed out

---
 test/scale/connectivity/test-connectivity.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/scale/connectivity/test-connectivity.sh b/test/scale/connectivity/test-connectivity.sh
index 85f72f054b..b7db9ca96c 100755
--- a/test/scale/connectivity/test-connectivity.sh
+++ b/test/scale/connectivity/test-connectivity.sh
@@ -256,7 +256,7 @@ done
 
 low=0
 if [[ $prevTryDate -gt $connectivityStartDate ]]; then
-    low=$(( `date +%s` - $prevTryDate - $CONNECTIVITY_SLEEP ))
+    low=$(( $prevTryDate - $connectivityStartDate - $CONNECTIVITY_SLEEP ))
 fi
 high=$(( `date +%s` - $connectivityStartDate ))
 echo "SUCCESS: all initial connectivity tests passed. Took between $low and $high seconds to succeed"
@@ -310,7 +310,7 @@ done
 
 low=0
 if [[ $prevTryDate -gt $netpolStartDate ]]; then
-    low=$(( `date +%s` - $prevTryDate - $NETPOL_SLEEP ))
+    low=$(( $prevTryDate - $connectivityStartDate - $NETPOL_SLEEP ))
 fi
 high=$(( `date +%s` - $netpolStartDate ))
 echo "SUCCESS: all connectivity tests passed after adding allow-pinger NetworkPolicy. Took between $low and $high seconds to take effect"

From 117272c8ac13260c0de5f42b0186b2d41ab0b3f3 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 16:54:03 -0700
Subject: [PATCH 04/20] unapplied netpols and delete chaos

---
 test/scale/connectivity/test-connectivity.sh  |   4 +-
 test/scale/templates/kwok-deployment.yaml     |   1 +
 test/scale/templates/real-deployment.yaml     |   1 +
 .../templates/unapplied-networkpolicy.yaml    |  22 +++
 test/scale/test-scale.sh                      | 157 ++++++++++++++++--
 5 files changed, 172 insertions(+), 13 deletions(-)
 create mode 100644 test/scale/templates/unapplied-networkpolicy.yaml

diff --git a/test/scale/connectivity/test-connectivity.sh b/test/scale/connectivity/test-connectivity.sh
index b7db9ca96c..f6268f0c71 100755
--- a/test/scale/connectivity/test-connectivity.sh
+++ b/test/scale/connectivity/test-connectivity.sh
@@ -123,7 +123,7 @@ echo "STARTING CONNECTIVITY TEST at $startDate"
 
 ## GET SCALE PODS
 echo "getting scale Pods..."
-scalePodNameIPs=(`kubectl $KUBECONFIG_ARG get pods -n scale-test --field-selector=status.phase==Running -o jsonpath='{range .items[*]}{@.metadata.name}{","}{@.status.podIP}{" "}{end}'`)
+scalePodNameIPs=(`kubectl $KUBECONFIG_ARG get pods -n scale-test --field-selector=status.phase==Running -l is-real="true" -o jsonpath='{range .items[*]}{@.metadata.name}{","}{@.status.podIP}{" "}{end}'`)
 scalePods=()
 scalePodIPs=()
 for nameIP in "${scalePodNameIPs[@]}"; do
@@ -131,8 +131,6 @@ for nameIP in "${scalePodNameIPs[@]}"; do
     name=${nameIP[0]}
     ip=${nameIP[1]}
 
-    echo $name | grep real-dep || continue
-
     echo "scale Pod: $name, IP: $ip"
 
     if [[ -z $name || -z $ip ]]; then
diff --git a/test/scale/templates/kwok-deployment.yaml b/test/scale/templates/kwok-deployment.yaml
index 70ef60d5c2..af145cdcb8 100644
--- a/test/scale/templates/kwok-deployment.yaml
+++ b/test/scale/templates/kwok-deployment.yaml
@@ -5,6 +5,7 @@ metadata:
   namespace: scale-test
   labels:
     app: scale-test
+    is-kwok: "true"
 spec:
   replicas: TEMP_REPLICAS
   selector:
diff --git a/test/scale/templates/real-deployment.yaml b/test/scale/templates/real-deployment.yaml
index 38385eda49..cc930a1ce6 100644
--- a/test/scale/templates/real-deployment.yaml
+++ b/test/scale/templates/real-deployment.yaml
@@ -5,6 +5,7 @@ metadata:
   namespace: scale-test
   labels:
     app: scale-test
+    is-real: "true"
 spec:
   replicas: TEMP_REPLICAS
   selector:
diff --git a/test/scale/templates/unapplied-networkpolicy.yaml b/test/scale/templates/unapplied-networkpolicy.yaml
new file mode 100644
index 0000000000..812fb38944
--- /dev/null
+++ b/test/scale/templates/unapplied-networkpolicy.yaml
@@ -0,0 +1,22 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: TEMP_NAME
+  namespace: scale-test
+spec:
+  podSelector:
+    matchLabels:
+      non-existent-key: val
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          non-existent-key: val
+  egress:
+  - to:
+    - podSelector:
+        matchLabels:
+          non-existent-key: val
+  policyTypes:
+  - Ingress
+  - Egress
diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 981d505348..3a7ef5a28e 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -3,7 +3,8 @@ set -e
 
 printHelp() {
     cat <<EOF
-./test-scale.sh --max-kwok-pods-per-node=<int> --num-kwok-deployments=<int> --num-kwok-replicas=<int> --max-real-pods-per-node=<int> --num-real-deployments=<int> --num-real-replicas=<int> --num-network-policies=<int> --num-unique-labels-per-pod=<int> --num-unique-labels-per-deployment=<int> --num-shared-labels-per-pod=<int> [--kubeconfig=<path>] [--restart-npm] [--debug-exit-after-print-counts] [--debug-exit-after-generation]
+./test-scale.sh --max-kwok-pods-per-node=<int> --num-kwok-deployments=<int> --num-kwok-replicas=<int> --max-real-pods-per-node=<int> --num-real-deployments=<int> --num-real-replicas=<int> --num-network-policies=<int> --num-unapplied-network-policies=<int> --num-unique-labels-per-pod=<int> --num-unique-labels-per-deployment=<int> --num-shared-labels-per-pod=<int> [--kubeconfig=<path>] [--restart-npm] [--debug-exit-after-print-counts] [--debug-exit-after-generation]
+(more optional parameters at end of this message)
 
 Scales the number of Pods, Pod labels, and NetworkPolicies in a cluster.
 Uses KWOK to create fake nodes and fake pods as needed.
@@ -28,6 +29,7 @@ REQUIRED PARAMETERS:
     --num-real-deployments                deployments scheduled on nodes labeled with scale-test=true
     --num-real-replicas                   per deployment
     --num-network-policies                NetPols applied to every Pod
+    --num-unapplied-network-policies      NetPols that do not target any Pods
     --num-unique-labels-per-pod           creates labels specific to each Pod. Creates numTotalPods*numUniqueLabelsPerPod distinct labels. In Cilium, a value >= 1 results in every Pod having a unique identity (not recommended for scale)
     --num-unique-labels-per-deployment    create labels shared between replicas of a deployment. Creates numTotalDeployments*numUniqueLabelsPerDeployment distinct labels
     --num-shared-labels-per-pod           create labels shared between all Pods. Creates numSharedLabelsPerPod distinct labels. Must be >= 3 if numNetworkPolicies > 0 because of the way we generate network policies
@@ -37,6 +39,19 @@ OPTIONAL PARAMETERS:
     --restart-npm                         make sure NPM exists and restart it before running scale test
     --debug-exit-after-print-counts       skip scale test. Just print out counts of things to be created and counts of IPSets/ACLs that NPM would create
     --debug-exit-after-generation         skip scale test. Exit after generating templates
+
+OPTIONAL PARAMETERS TO TEST DELETION:
+    --sleep-after-creation=<int>          seconds to sleep after creating everything. Default is 0
+    --delete-kwok-pods=<int>              delete and readd the specified number of fake Pods
+    --delete-real-pods=<int>              delete and readd the specified number of real Pods
+    --delete-pods-interval=<int>          seconds to wait after deleting Pods. Default is 60
+    --delete-pods-times=<int>             number of times to delete and readd. Default is 1
+    --delete-labels                       delete and readd shared labels from all Pods
+    --delete-labels-interval=<int>        seconds to wait after deleting or readding. Default is 60
+    --delete-labels-times=<int>           number of times to delete and readd. Default is 1
+    --delete-netpols                      delete and readd all NetworkPolicies
+    --delete-netpols-interval=<int>       seconds to wait after deleting or readding. Default is 60
+    --delete-netpols-times=<int>          number of times to delete and readd. Default is 1
 EOF
 }
 
@@ -68,6 +83,9 @@ while [[ $# -gt 0 ]]; do
         --num-network-policies=*)
             numNetworkPolicies="${1#*=}"
             ;;
+        --num-unapplied-network-policies=*)
+            numUnappliedNetworkPolicies="${1#*=}"
+            ;;
         --num-unique-labels-per-pod=*)
             numUniqueLabelsPerPod="${1#*=}"
             ;;
@@ -95,6 +113,39 @@ while [[ $# -gt 0 ]]; do
         --debug-exit-after-generation)
             DEBUG_EXIT_AFTER_GENERATION=true
             ;;
+        --sleep-after-creation=*)
+            sleepAfterCreation="${1#*=}"
+            ;;
+        --delete-kwok-pods=*)
+            deleteKwokPods="${1#*=}"
+            ;;
+        --delete-real-pods=*)
+            deleteRealPods="${1#*=}"
+            ;;
+        --delete-pods-interval=*)
+            deletePodsInterval="${1#*=}"
+            ;;
+        --delete-pods-times=*)
+            deletePodsTimes="${1#*=}"
+            ;;
+        --delete-labels)
+            deleteLabels=true
+            ;;
+        --delete-labels-interval=*)
+            deleteLabelsInterval="${1#*=}"
+            ;;
+        --delete-labels-times=*)
+            deleteLabelsTimes="${1#*=}"
+            ;;
+        --delete-netpols)
+            deleteNetpols=true
+            ;;
+        --delete-netpols-interval=*)
+            deleteNetpolsInterval="${1#*=}"
+            ;;
+        --delete-netpols-times=*)
+            deleteNetpolsTimes="${1#*=}"
+            ;;
         *)
             echo "ERROR: unknown parameter $1. Make sure you're using '--key=value' for parameters with values"
             exit 1
@@ -103,7 +154,7 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-if [[ -z $maxKwokPodsPerNode || -z $numKwokDeployments || -z $numKwokReplicas || -z $maxRealPodsPerNode || -z $numRealDeployments || -z $numRealReplicas || -z $numNetworkPolicies || -z $numUniqueLabelsPerPod || -z $numUniqueLabelsPerDeployment || -z $numSharedLabelsPerPod ]]; then
+if [[ -z $maxKwokPodsPerNode || -z $numKwokDeployments || -z $numKwokReplicas || -z $maxRealPodsPerNode || -z $numRealDeployments || -z $numRealReplicas || -z $numNetworkPolicies || -z $numUnappliedNetworkPolicies || -z $numUniqueLabelsPerPod || -z $numUniqueLabelsPerDeployment || -z $numSharedLabelsPerPod ]]; then
     echo "ERROR: missing required parameter. Check --help for usage"
     exit 1
 fi
@@ -113,6 +164,13 @@ if [[ $numNetworkPolicies -gt 0 && $numSharedLabelsPerPod -lt 3 ]]; then
     exit 1
 fi
 
+if [[ -z $deletePodsInterval ]]; then deletePodsInterval=60; fi
+if [[ -z $deletePodsTimes ]]; then deletePodsTimes=1; fi
+if [[ -z $deleteLabelsInterval ]]; then deleteLabelsInterval=60; fi
+if [[ -z $deleteLabelsTimes ]]; then deleteLabelsTimes=1; fi
+if [[ -z $deleteNetpolsInterval ]]; then deleteNetpolsInterval=60; fi
+if [[ -z $deleteNetpolsTimes ]]; then deleteNetpolsTimes=1; fi
+
 ## CALCULATIONS
 numKwokPods=$(( $numKwokDeployments * $numKwokReplicas ))
 numKwokNodes=$(( ($numKwokPods + $maxKwokPodsPerNode - 1) / $maxKwokPodsPerNode))
@@ -125,10 +183,11 @@ numTotalPods=$(( $numKwokPods + $numRealPods ))
 numACLsAddedByNPM=$(( 6 * $numNetworkPolicies ))
 # IPSet/member counts can be slight underestimates if there are more than one template-hash labels
 # 4 basic IPSets are [ns-scale-test,kubernetes.io/metadata.name:scale-test,template-hash:xxxx,app:scale-test]
-numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment ))
+# also have [is-real, is-real:true, is-kwok, is-kwok:true]
+numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*($numUniqueLabelsPerPod+1) + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment ))
 # 3 basic members are [all-ns,kubernetes.io/metadata.name,kubernetes.io/metadata.name:scale-test]
 # 5*pods members go to [ns-scale-test,kubernetes.io/metadata.name:scale-test,template-hash:xxxx,app:scale-test]
-numIPSetMembersAddedByNPM=$(( 3 + $numTotalPods*(5 + 2*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod) + 2*($numKwokPods+$numRealPods)*$numUniqueLabelsPerDeployment ))
+numIPSetMembersAddedByNPM=$(( 3 + $numTotalPods*(5 + 2*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod) + 2*($numKwokPods+$numRealPods)*$numUniqueLabelsPerDeployment + 2*$numKwokPods + 2*$numRealPods ))
 
 ## PRINT OUT COUNTS
 cat <<EOF
@@ -164,7 +223,8 @@ fi
 ## FILE SETUP
 echo "Cleaning up generated/ directory..."
 test -d generated && rm -rf generated/
-mkdir -p generated/networkpolicies/
+mkdir -p generated/networkpolicies/applied
+mkdir -p generated/networkpolicies/unapplied
 mkdir -p generated/kwok-nodes
 mkdir -p generated/deployments/real/
 mkdir -p generated/deployments/kwok/
@@ -209,20 +269,26 @@ generateDeployments $numRealDeployments $numRealReplicas real
 for j in $(seq 1 $numNetworkPolicies); do
     valNum=$j
     i=`printf "%05d" $j`
-    sed "s/TEMP_NAME/policy-$i/g" templates/networkpolicy.yaml > generated/networkpolicies/policy-$i.yaml
+    fileName=generated/networkpolicies/applied/policy-$i.yaml
+    sed "s/TEMP_NAME/policy-$i/g" templates/networkpolicy.yaml > $fileName
     if [[ $valNum -ge $(( numSharedLabelsPerPod - 2 )) ]]; then
         valNum=$(( $numSharedLabelsPerPod - 2 ))
     fi
     k=`printf "%05d" $valNum`
-    sed -i "s/TEMP_LABEL_NAME/shared-lab-$k/g" generated/networkpolicies/policy-$i.yaml
+    sed -i "s/TEMP_LABEL_NAME/shared-lab-$k/g" $fileName
 
     ingressNum=$(( $valNum + 1 ))
     k=`printf "%05d" $ingressNum`
-    sed -i "s/TEMP_INGRESS_NAME/shared-lab-$k/g" generated/networkpolicies/policy-$i.yaml
+    sed -i "s/TEMP_INGRESS_NAME/shared-lab-$k/g" $fileName
 
     egressNum=$(( $valNum + 2 ))
     k=`printf "%05d" $ingressNum`
-    sed -i "s/TEMP_EGRESS_NAME/shared-lab-$k/g" generated/networkpolicies/policy-$i.yaml
+    sed -i "s/TEMP_EGRESS_NAME/shared-lab-$k/g" $fileName
+done
+
+for j in $(seq 1 $numUnappliedNetworkPolicies ); do
+    i=`printf "%05d" $j`
+    sed "s/TEMP_NAME/unapplied-policy-$i/g" templates/unapplied-networkpolicy.yaml > generated/networkpolicies/unapplied/unapplied-policy-$i.yaml
 done
 
 for i in $(seq -f "%05g" 1 $numKwokNodes); do
@@ -320,9 +386,80 @@ if [[ $numUniqueLabelsPerPod -gt 0 ]]; then
 fi
 
 set -x
-kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/
+kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/unapplied
+kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/applied
+# wait for all pods to run
+kubectl wait --for=condition=Ready pods -n scale-test --all --timeout=15m
 set +x
 
 echo
 echo "FINISHED at $(date -u). Had started at $startDate."
 echo
+
+echo "performing deletions if configured..."
+
+if [[ $sleepAfterCreation != "" ]]; then
+    echo "sleeping $sleepAfterCreation seconds after creation..."
+    sleep $sleepAfterCreation
+fi
+
+if [[ $deleteLabels == true && $numSharedLabelsPerPod -gt 2 ]]; then
+    echo "deleting labels..."
+    for i in $(seq 1 $deleteLabelsTimes); do
+        echo "deleting labels. round $i/$deleteLabelsTimes..."
+        set -x
+        kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001- shared-lab-00002- shared-lab-00003-
+        set +x
+        echo "sleeping $deleteLabelsSleep seconds after deleting label (round $i/$deleteLabelsTimes)..."
+        sleep $deleteLabelsSleep
+        
+        echo "re-adding labels. round $i/$deleteLabelsTimes..."
+        set -x
+        kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001=val shared-lab-00002=val shared-lab-00003=val
+        set +x
+        echo "sleeping $deleteLabelsSleep seconds after readding label (end of round $i/$deleteLabelsTimes)..."
+        sleep $deleteLabelsSleep
+    done
+fi
+
+if [[ $deleteNetpols == true ]]; then
+    echo "deleting network policies..."
+    for i in $(seq 1 $deleteNetpolsTimes); do
+        echo "deleting network policies. round $i/$deleteNetpolsTimes..."
+        set -x
+        kubectl $KUBECONFIG_ARG delete netpol -n scale-test --all
+        set +x
+        echo "sleeping $deleteNetpolsSleep seconds after deleting network policies (round $i/$deleteNetpolsTimes)..."
+        sleep $deleteNetpolsSleep
+        
+        echo "re-adding network policies. round $i/$deleteNetpolsTimes..."
+        set -x
+        kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/unapplied
+        kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/applied
+        set +x
+        echo "sleeping $deleteNetpolsSleep seconds after readding network policies (end of round $i/$deleteNetpolsTimes)..."
+        sleep $deleteNetpolsSleep
+    done
+fi
+
+if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != "" && $deleteRealPods -gt 0) ]]; then
+    for i in $(seq 1 $deletePodsTimes); do
+        if [[ $deleteKwokPods != "" && $deleteKwokPods -gt 0 && $numKwokPods -gt 0 ]]; then
+            echo "deleting kwok pods. round $i/$deletePodsTimes..."
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-kwok="true" | shuf -n $deleteKwokPods | | awk '{print $1}' | tr '\n' ' '`
+            set -x
+            kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
+            set +x
+        fi
+
+        if [[ $deleteRealPods != "" &&  && $deleteRealPods -gt 0 && $numRealPods -gt 0 ]]; then
+            echo "deleting real pods. round $i/$deletePodsTimes..."
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | shuf -n $deleteRealPods | | awk '{print $1}' | tr '\n' ' '`
+            set -x
+            kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
+            set +x
+        fi
+
+        echo "sleeping $deletePodsSleep seconds after deleting pods (end of round $i/$deletePodsTimes)..."
+    done
+fi

From 6630512d61ce48766974ccbeaf6ba1dfb836514b Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:11:31 -0700
Subject: [PATCH 05/20] print out new args

---
 test/scale/test-scale.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 3a7ef5a28e..f69256e646 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -201,6 +201,19 @@ numSharedLabelsPerPod=$numSharedLabelsPerPod
 numUniqueLabelsPerPod=$numUniqueLabelsPerPod
 numUniqueLabelsPerDeployment=$numUniqueLabelsPerDeployment
 numNetworkPolicies=$numNetworkPolicies
+numUnappliedNetworkPolicies=$numUnappliedNetworkPolicies
+
+Delete arguments (optional):
+deleteKwokPods=$deleteKwokPods
+deleteRealPods=$deleteRealPods
+deletePodsInterval=$deletePodsInterval
+deletePodsTimes=$deletePodsTimes
+deleteLabels=$deleteLabels
+deleteLabelsInterval=$deleteLabelsInterval
+deleteLabelsTimes=$deleteLabelsTimes
+deleteNetpols=$deleteNetpols
+deleteNetpolsInterval=$deleteNetpolsInterval
+deleteNetpolsTimes=$deleteNetpolsTimes
 
 The following will be created:
 kwok Nodes: $numKwokNodes

From d5a94d7ad462b69ea9b313b172f162ad65223036 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:19:32 -0700
Subject: [PATCH 06/20] simplify kwok install and run script

---
 test/scale/install-kwok.sh |  5 +++++
 test/scale/run-kwok.sh     | 34 +++++-----------------------------
 2 files changed, 10 insertions(+), 29 deletions(-)
 create mode 100644 test/scale/install-kwok.sh

diff --git a/test/scale/install-kwok.sh b/test/scale/install-kwok.sh
new file mode 100644
index 0000000000..46618258d6
--- /dev/null
+++ b/test/scale/install-kwok.sh
@@ -0,0 +1,5 @@
+KWOK_REPO=kubernetes-sigs/kwok
+KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
+wget -O kwok -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_LATEST_RELEASE}/kwok-$(go env GOOS)-$(go env GOARCH)"
+chmod +x kwok
+sudo mv kwok /usr/local/bin/kwok
diff --git a/test/scale/run-kwok.sh b/test/scale/run-kwok.sh
index f1118ae320..5ff69a0ddf 100755
--- a/test/scale/run-kwok.sh
+++ b/test/scale/run-kwok.sh
@@ -1,33 +1,9 @@
-######################################################################################
-# This script is used to schedule kwok nodes/pods and maintain kwok node heartbeats. #
-######################################################################################
-INSTALL_KWOK=false
-# KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
-KWOK_VERSION=${KWOK_LATEST_RELEASE:-"v0.1.1"}
-# kubeconfig arg doesn't seem to work for kwok. It seems to just use current context of the default kubeconfig.
+###############################################################
+# Schedule kwok nodes/pods and maintain kwok node heartbeats. #
+# Install kwok via ./install-kwok.sh                          #
+###############################################################
 
-# specify kubeconfig file as first arg if you want
-if [[ $1 != "" ]]; then
-    file=$1
-    test -f $file || {
-        echo "ERROR: KUBECONFIG=$file does not exist"
-        exit 1
-    }
-
-    KUBECONFIG_ARG="--kubeconfig $file"
-fi
-
-if [[ INSTALL_KWOK == true ]]; then
-    wget -O kwokctl -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_VERSION}/kwokctl-$(go env GOOS)-$(go env GOARCH)"
-    chmod +x kwokctl
-    sudo mv kwokctl /usr/local/bin/kwokctl
-
-    wget -O kwok -c "https://github.com/kubernetes-sigs/kwok/releases/download/${KWOK_VERSION}/kwok-$(go env GOOS)-$(go env GOARCH)"
-    chmod +x kwok
-    sudo mv kwok /usr/local/bin/kwok
-fi
-
-kwok $KUBECONFIG_ARG \
+kwok --kubeconfig ~/.kube/config \
     --cidr=155.0.0.0/16 \
     --node-ip=155.0.0.1 \
     --manage-all-nodes=false \

From 3c201e329c31d30fd6d662e134fcf23c56d76558 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:24:57 -0700
Subject: [PATCH 07/20] fix typo

---
 test/scale/test-scale.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index f69256e646..78de98fbd2 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -465,7 +465,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
             set +x
         fi
 
-        if [[ $deleteRealPods != "" &&  && $deleteRealPods -gt 0 && $numRealPods -gt 0 ]]; then
+        if [[ $deleteRealPods != "" && $deleteRealPods -gt 0 && $numRealPods -gt 0 ]]; then
             echo "deleting real pods. round $i/$deletePodsTimes..."
             pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | shuf -n $deleteRealPods | | awk '{print $1}' | tr '\n' ' '`
             set -x

From 0dc7074e7acafcf330aef66de3cc3e9057e06f27 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:36:54 -0700
Subject: [PATCH 08/20] example run

---
 test/scale/README.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/test/scale/README.md b/test/scale/README.md
index 33eb3636f6..1e38b8498a 100644
--- a/test/scale/README.md
+++ b/test/scale/README.md
@@ -13,3 +13,28 @@ This saves us from:
 2. If making KWOK Pods, run `run-kwok.sh` in the background.
 3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods.
 4. Test connectivity with `connectivity/test-connectivity.sh`.
+
+### Example Run
+```
+./test-scale.sh --max-kwok-pods-per-node=50 \
+    --num-kwok-deployments=10 \
+    --num-kwok-replicas=1 \
+    --max-real-pods-per-node=30 \
+    --num-real-deployments=5 \
+    --num-real-replicas=2 \
+    --num-network-policies=1 \
+    --num-unapplied-network-policies=10 \
+    --num-unique-labels-per-pod=2 \
+    --num-unique-labels-per-deployment=2 \
+    --num-shared-labels-per-pod=10 \
+    --delete-kwok-pods=10 \
+    --delete-real-pods=5 \
+    --delete-pods-interval=120 \
+    --delete-pods-times=2 \
+    --delete-labels \
+    --delete-labels-interval=30 \
+    --delete-labels-times=2 \
+    --delete-netpols \
+    --delete-netpols-interval=0 \
+    --delete-netpols-times=1 | tee scale.out
+```

From b97e4ee3f53dd42301bb44e467a841ea157f4592 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:45:09 -0700
Subject: [PATCH 09/20] fix sleep and num ipsets

---
 test/scale/test-scale.sh | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 78de98fbd2..7fef3281ca 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -183,8 +183,19 @@ numTotalPods=$(( $numKwokPods + $numRealPods ))
 numACLsAddedByNPM=$(( 6 * $numNetworkPolicies ))
 # IPSet/member counts can be slight underestimates if there are more than one template-hash labels
 # 4 basic IPSets are [ns-scale-test,kubernetes.io/metadata.name:scale-test,template-hash:xxxx,app:scale-test]
-# also have [is-real, is-real:true, is-kwok, is-kwok:true]
-numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*($numUniqueLabelsPerPod+1) + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment ))
+# for deployments, have [is-real, is-real:true, is-kwok, is-kwok:true]
+# for unapplied netpols, have [non-existent-key, non-existent-key:val]
+extraIPSets=0
+if [[ $numUnappliedNetworkPolicies -gt 0 ]]; then
+    extraIPSets=$(( $extraIPSets + 2 ))
+fi
+if [[ $numKwokPods -gt 0 ]]; then
+    extraIPSets=$(( $extraIPSets + 2 ))
+fi
+if [[ $numRealPods -gt 0 ]]; then
+    extraIPSets=$(( $extraIPSets + 2 ))
+fi
+numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment + $extraIPSets ))
 # 3 basic members are [all-ns,kubernetes.io/metadata.name,kubernetes.io/metadata.name:scale-test]
 # 5*pods members go to [ns-scale-test,kubernetes.io/metadata.name:scale-test,template-hash:xxxx,app:scale-test]
 numIPSetMembersAddedByNPM=$(( 3 + $numTotalPods*(5 + 2*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod) + 2*($numKwokPods+$numRealPods)*$numUniqueLabelsPerDeployment + 2*$numKwokPods + 2*$numRealPods ))
@@ -423,15 +434,15 @@ if [[ $deleteLabels == true && $numSharedLabelsPerPod -gt 2 ]]; then
         set -x
         kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001- shared-lab-00002- shared-lab-00003-
         set +x
-        echo "sleeping $deleteLabelsSleep seconds after deleting label (round $i/$deleteLabelsTimes)..."
-        sleep $deleteLabelsSleep
+        echo "sleeping $deleteLabelsInterval seconds after deleting label (round $i/$deleteLabelsTimes)..."
+        sleep $deleteLabelsInterval
         
         echo "re-adding labels. round $i/$deleteLabelsTimes..."
         set -x
         kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001=val shared-lab-00002=val shared-lab-00003=val
         set +x
-        echo "sleeping $deleteLabelsSleep seconds after readding label (end of round $i/$deleteLabelsTimes)..."
-        sleep $deleteLabelsSleep
+        echo "sleeping $deleteLabelsInterval seconds after readding label (end of round $i/$deleteLabelsTimes)..."
+        sleep $deleteLabelsInterval
     done
 fi
 

From d8adfc3966afcf6c440cfc1409822809fcc45a20 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:51:56 -0700
Subject: [PATCH 10/20] better run-kwok.sh

---
 test/scale/run-kwok.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/test/scale/run-kwok.sh b/test/scale/run-kwok.sh
index 5ff69a0ddf..320758933b 100755
--- a/test/scale/run-kwok.sh
+++ b/test/scale/run-kwok.sh
@@ -1,9 +1,21 @@
 ###############################################################
 # Schedule kwok nodes/pods and maintain kwok node heartbeats. #
-# Install kwok via ./install-kwok.sh                          #
 ###############################################################
+# can pass kubeconfig as first arg
+if [[ -z $1 ]]; then
+    kubeconfigFile=$1
+else
+    kubeconfigFile=~/.kube/config
+fi
+echo "using kubeconfig $kubeconfigFile"
 
-kwok --kubeconfig ~/.kube/config \
+which kwok || {
+    echo "ERROR: kwok not found. Install via ./install-kwok.sh"
+    exit 1
+}
+
+set -x
+kwok --kubeconfig $kubeconfigFile \
     --cidr=155.0.0.0/16 \
     --node-ip=155.0.0.1 \
     --manage-all-nodes=false \

From e745c0e615bbe990a66092325539f8d8f62790ce Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:54:09 -0700
Subject: [PATCH 11/20] fix sleep

---
 test/scale/test-scale.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 7fef3281ca..da93f3500c 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -453,16 +453,16 @@ if [[ $deleteNetpols == true ]]; then
         set -x
         kubectl $KUBECONFIG_ARG delete netpol -n scale-test --all
         set +x
-        echo "sleeping $deleteNetpolsSleep seconds after deleting network policies (round $i/$deleteNetpolsTimes)..."
-        sleep $deleteNetpolsSleep
+        echo "sleeping $deleteNetpolsInterval seconds after deleting network policies (round $i/$deleteNetpolsTimes)..."
+        sleep $deleteNetpolsInterval
         
         echo "re-adding network policies. round $i/$deleteNetpolsTimes..."
         set -x
         kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/unapplied
         kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/applied
         set +x
-        echo "sleeping $deleteNetpolsSleep seconds after readding network policies (end of round $i/$deleteNetpolsTimes)..."
-        sleep $deleteNetpolsSleep
+        echo "sleeping $deleteNetpolsInterval seconds after readding network policies (end of round $i/$deleteNetpolsTimes)..."
+        sleep $deleteNetpolsInterval
     done
 fi
 
@@ -484,6 +484,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
             set +x
         fi
 
-        echo "sleeping $deletePodsSleep seconds after deleting pods (end of round $i/$deletePodsTimes)..."
+        echo "sleeping $deletePodsInterval seconds after deleting pods (end of round $i/$deletePodsTimes)..."
+        sleep $deletePodsInterval
     done
 fi

From 71af93debca614ac4c9d9a311cd0c4318b049b00 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 18:55:00 -0700
Subject: [PATCH 12/20] fix run kwok

---
 test/scale/run-kwok.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/scale/run-kwok.sh b/test/scale/run-kwok.sh
index 320758933b..77876fb916 100755
--- a/test/scale/run-kwok.sh
+++ b/test/scale/run-kwok.sh
@@ -3,9 +3,9 @@
 ###############################################################
 # can pass kubeconfig as first arg
 if [[ -z $1 ]]; then
-    kubeconfigFile=$1
-else
     kubeconfigFile=~/.kube/config
+else
+    kubeconfigFile=$1
 fi
 echo "using kubeconfig $kubeconfigFile"
 

From dc5510c15250ceaeda2a5546b8548f651f62ddab Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Wed, 12 Apr 2023 19:07:00 -0700
Subject: [PATCH 13/20] fix pipe

---
 test/scale/test-scale.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index da93f3500c..8d2cfe8c33 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -470,7 +470,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
     for i in $(seq 1 $deletePodsTimes); do
         if [[ $deleteKwokPods != "" && $deleteKwokPods -gt 0 && $numKwokPods -gt 0 ]]; then
             echo "deleting kwok pods. round $i/$deletePodsTimes..."
-            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-kwok="true" | shuf -n $deleteKwokPods | | awk '{print $1}' | tr '\n' ' '`
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-kwok="true" | shuf -n $deleteKwokPods | awk '{print $1}' | tr '\n' ' '`
             set -x
             kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
             set +x
@@ -478,7 +478,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
 
         if [[ $deleteRealPods != "" && $deleteRealPods -gt 0 && $numRealPods -gt 0 ]]; then
             echo "deleting real pods. round $i/$deletePodsTimes..."
-            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | shuf -n $deleteRealPods | | awk '{print $1}' | tr '\n' ' '`
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | shuf -n $deleteRealPods | awk '{print $1}' | tr '\n' ' '`
             set -x
             kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
             set +x

From 047bab9eb653629a07690e99585b298be0e5da27 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 09:40:38 -0700
Subject: [PATCH 14/20] fix labels on pods

---
 test/scale/templates/kwok-deployment.yaml | 6 ++++--
 test/scale/templates/real-deployment.yaml | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/test/scale/templates/kwok-deployment.yaml b/test/scale/templates/kwok-deployment.yaml
index af145cdcb8..4491e5496b 100644
--- a/test/scale/templates/kwok-deployment.yaml
+++ b/test/scale/templates/kwok-deployment.yaml
@@ -10,11 +10,13 @@ spec:
   replicas: TEMP_REPLICAS
   selector:
     matchLabels:
-      app: scale-testOTHER_LABELS_6_SPACES
+      app: scale-test
+      is-kwok: "true"OTHER_LABELS_6_SPACES
   template:
     metadata:
       labels:
-        app: scale-testOTHER_LABELS_8_SPACES
+        app: scale-test
+        is-kwok: "true"OTHER_LABELS_8_SPACES
     spec:
       affinity:
         nodeAffinity:
diff --git a/test/scale/templates/real-deployment.yaml b/test/scale/templates/real-deployment.yaml
index cc930a1ce6..9605dbf980 100644
--- a/test/scale/templates/real-deployment.yaml
+++ b/test/scale/templates/real-deployment.yaml
@@ -10,11 +10,13 @@ spec:
   replicas: TEMP_REPLICAS
   selector:
     matchLabels:
-      app: scale-testOTHER_LABELS_6_SPACES
+      app: scale-test
+      is-real: "true"OTHER_LABELS_6_SPACES
   template:
     metadata:
       labels:
-        app: scale-testOTHER_LABELS_8_SPACES
+        app: scale-test
+        is-real: "true"OTHER_LABELS_8_SPACES
     spec:
       nodeSelector:
         scale-test: "true"

From 718a986bcbffba1b82da4db43d9e9a9d7b5b988c Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 09:50:51 -0700
Subject: [PATCH 15/20] fix delete

---
 test/scale/test-scale.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 8d2cfe8c33..7efac60f65 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -470,7 +470,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
     for i in $(seq 1 $deletePodsTimes); do
         if [[ $deleteKwokPods != "" && $deleteKwokPods -gt 0 && $numKwokPods -gt 0 ]]; then
             echo "deleting kwok pods. round $i/$deletePodsTimes..."
-            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-kwok="true" | shuf -n $deleteKwokPods | awk '{print $1}' | tr '\n' ' '`
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-kwok="true" | grep -v NAME | shuf -n $deleteKwokPods | awk '{print $1}' | tr '\n' ' '`
             set -x
             kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
             set +x
@@ -478,7 +478,7 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
 
         if [[ $deleteRealPods != "" && $deleteRealPods -gt 0 && $numRealPods -gt 0 ]]; then
             echo "deleting real pods. round $i/$deletePodsTimes..."
-            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | shuf -n $deleteRealPods | awk '{print $1}' | tr '\n' ' '`
+            pods=`kubectl $KUBECONFIG_ARG get pods -n scale-test -l is-real="true" | grep -v NAME | shuf -n $deleteRealPods | awk '{print $1}' | tr '\n' ' '`
             set -x
             kubectl $KUBECONFIG_ARG delete pods -n scale-test $pods
             set +x

From fa9738d5c65f4d78cac7a9a27c054bb3f7322cf2 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 10:19:20 -0700
Subject: [PATCH 16/20] add connectivity to readme

---
 test/scale/README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/scale/README.md b/test/scale/README.md
index 1e38b8498a..452010d297 100644
--- a/test/scale/README.md
+++ b/test/scale/README.md
@@ -14,7 +14,7 @@ This saves us from:
 3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods.
 4. Test connectivity with `connectivity/test-connectivity.sh`.
 
-### Example Run
+### Example Runs
 ```
 ./test-scale.sh --max-kwok-pods-per-node=50 \
     --num-kwok-deployments=10 \
@@ -36,5 +36,11 @@ This saves us from:
     --delete-labels-times=2 \
     --delete-netpols \
     --delete-netpols-interval=0 \
-    --delete-netpols-times=1 | tee scale.out
+    --delete-netpols-times=1
+```
+
+```
+./test-connectivity --num-scale-pods-to-verify=10 \
+    --max-wait-for-initial-connectivity=600 \
+    --max-wait-after-adding-netpol=120
 ```

From 9a149cbb4918fea89d60c086caec512113ac0b9c Mon Sep 17 00:00:00 2001
From: Hunter Gregory <42728408+huntergregory@users.noreply.github.com>
Date: Thu, 13 Apr 2023 10:24:53 -0700
Subject: [PATCH 17/20] Update README.md

---
 test/scale/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scale/README.md b/test/scale/README.md
index 452010d297..8945bcffd2 100644
--- a/test/scale/README.md
+++ b/test/scale/README.md
@@ -11,7 +11,7 @@ This saves us from:
 ## Usage
 1. Create AKS cluster with `--uptime-sla` and create any nodepools.
 2. If making KWOK Pods, run `run-kwok.sh` in the background.
-3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods.
+3. Scale with `test-scale.sh`. Specify number of Deployments, Pod replicas, NetworkPolicies, and labels for Pods. Can also delete/re-add objects to cause churn.
 4. Test connectivity with `connectivity/test-connectivity.sh`.
 
 ### Example Runs

From 38aaa10b482151c4f2f25b22d709bdeba20db783 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 13:05:46 -0700
Subject: [PATCH 18/20] fix printing

---
 test/scale/test-scale.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index 7efac60f65..e7d6a8c102 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -417,7 +417,7 @@ kubectl wait --for=condition=Ready pods -n scale-test --all --timeout=15m
 set +x
 
 echo
-echo "FINISHED at $(date -u). Had started at $startDate."
+echo "done scaling at $(date -u). Had started at $startDate."
 echo
 
 echo "performing deletions if configured..."
@@ -434,14 +434,14 @@ if [[ $deleteLabels == true && $numSharedLabelsPerPod -gt 2 ]]; then
         set -x
         kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001- shared-lab-00002- shared-lab-00003-
         set +x
-        echo "sleeping $deleteLabelsInterval seconds after deleting label (round $i/$deleteLabelsTimes)..."
+        echo "sleeping $deleteLabelsInterval seconds after deleting labels (round $i/$deleteLabelsTimes)..."
         sleep $deleteLabelsInterval
         
         echo "re-adding labels. round $i/$deleteLabelsTimes..."
         set -x
         kubectl $KUBECONFIG_ARG label pods -n scale-test --all shared-lab-00001=val shared-lab-00002=val shared-lab-00003=val
         set +x
-        echo "sleeping $deleteLabelsInterval seconds after readding label (end of round $i/$deleteLabelsTimes)..."
+        echo "sleeping $deleteLabelsInterval seconds after readding labels (end of round $i/$deleteLabelsTimes)..."
         sleep $deleteLabelsInterval
     done
 fi
@@ -484,7 +484,14 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
             set +x
         fi
 
+        if [[ $i == $deletePodsTimes ]]; then
+            break
+        fi
         echo "sleeping $deletePodsInterval seconds after deleting pods (end of round $i/$deletePodsTimes)..."
         sleep $deletePodsInterval
     done
 fi
+
+echo
+echo "FINISHED at $(date -u). Had started at $startDate."
+echo

From 89818a2f1eaea26b5ffb428e1dcddc5305be72bd Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 13:55:03 -0700
Subject: [PATCH 19/20] wait for pods to come up after deleting

---
 test/scale/test-scale.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh
index e7d6a8c102..363acdd349 100755
--- a/test/scale/test-scale.sh
+++ b/test/scale/test-scale.sh
@@ -413,7 +413,7 @@ set -x
 kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/unapplied
 kubectl $KUBECONFIG_ARG apply -f generated/networkpolicies/applied
 # wait for all pods to run
-kubectl wait --for=condition=Ready pods -n scale-test --all --timeout=15m
+kubectl $KUBECONFIG_ARG wait --for=condition=Ready pods -n scale-test --all --timeout=15m
 set +x
 
 echo
@@ -484,6 +484,11 @@ if [[ ($deleteKwokPods != "" && $deleteKwokPods -gt 0) || ($deleteRealPods != ""
             set +x
         fi
 
+        sleep 5s
+        set -x
+        kubectl $KUBECONFIG_ARG wait --for=condition=Ready pods -n scale-test --all --timeout=15m
+        set +x
+
         if [[ $i == $deletePodsTimes ]]; then
             break
         fi

From b8f4fe52ec7bc4c5ef9b3ff1032f3a2595e396d3 Mon Sep 17 00:00:00 2001
From: Hunter Gregory <hunterlgregory@gmail.com>
Date: Thu, 13 Apr 2023 13:55:35 -0700
Subject: [PATCH 20/20] 'all' option

---
 test/scale/connectivity/test-connectivity.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/test/scale/connectivity/test-connectivity.sh b/test/scale/connectivity/test-connectivity.sh
index f6268f0c71..862f298c6e 100755
--- a/test/scale/connectivity/test-connectivity.sh
+++ b/test/scale/connectivity/test-connectivity.sh
@@ -10,7 +10,7 @@ NETPOL_SLEEP=5
 
 printHelp() {
     cat <<EOF
-./test-connectivity.sh --num-scale-pods-to-verify=<int> --max-wait-for-initial-connectivity=<int> --max-wait-after-adding-netpol=<int> [--kubeconfig=<path>]
+./test-connectivity.sh --num-scale-pods-to-verify=all|<int> --max-wait-for-initial-connectivity=<int> --max-wait-after-adding-netpol=<int> [--kubeconfig=<path>]
 
 Verifies that scale test Pods can connect to each other, but cannot connect to a new "pinger" Pod.
 Then, adds a NetworkPolicy to allow traffic between the scale test Pods and the "pinger" Pod, and verifies connectivity.
@@ -21,7 +21,7 @@ USAGE:
 3. Run this script
 
 REQUIRED PARAMETERS:
-    --num-scale-pods-to-verify=<int>             number of scale Pods to test. Will verify that each scale Pod can connect to each other [(N-1)^2 connections] and that each Scale Pod cannot connect to a "pinger" Pod [2N connection attempts with a 3-second timeout]
+    --num-scale-pods-to-verify=all|<int>         number of scale Pods to test. Will verify that each scale Pod can connect to each other [(N-1)^2 connections] and that each Scale Pod cannot connect to a "pinger" Pod [2N connection attempts with a 3-second timeout]
     --max-wait-for-initial-connectivity=<int>    maximum time in seconds to wait for initial connectivity after Pinger Pods are running
     --max-wait-after-adding-netpol=<int>         maximum time in seconds to wait for allowed connections after adding the allow-pinger NetworkPolicy
 
@@ -72,7 +72,7 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-if [[ -z $numScalePodsToVerify || -z $maxWaitAfterAddingNetpol ]]; then
+if [[ -z $numScalePodsToVerify || -z $maxWaitForInitialConnectivity || -z $maxWaitAfterAddingNetpol ]]; then
     echo "ERROR: missing required parameter. Check --help for usage"
     exit 6
 fi
@@ -122,6 +122,11 @@ startDate=`date -u`
 echo "STARTING CONNECTIVITY TEST at $startDate"
 
 ## GET SCALE PODS
+if [[ $numScalePodsToVerify == "all" ]]; then
+    echo "setting numScalePodsToVerify=9999 since 'all' was passed in"
+    numScalePodsToVerify=9999
+fi
+
 echo "getting scale Pods..."
 scalePodNameIPs=(`kubectl $KUBECONFIG_ARG get pods -n scale-test --field-selector=status.phase==Running -l is-real="true" -o jsonpath='{range .items[*]}{@.metadata.name}{","}{@.status.podIP}{" "}{end}'`)
 scalePods=()