From e75854e2229d8243075865b4be43cb518eabfb6c Mon Sep 17 00:00:00 2001 From: camrynl <31013536+camrynl@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:46:45 -0700 Subject: [PATCH 1/4] test: create nginx deployments in scale test --- test/scale/templates/label-nodes.sh | 32 +++++++++++++++++++ .../templates/real-nginx-deployment.yaml | 30 +++++++++++++++++ test/scale/test-scale.sh | 29 ++++++++++++++--- 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 test/scale/templates/label-nodes.sh create mode 100644 test/scale/templates/real-nginx-deployment.yaml diff --git a/test/scale/templates/label-nodes.sh b/test/scale/templates/label-nodes.sh new file mode 100644 index 0000000000..3bcc0623bc --- /dev/null +++ b/test/scale/templates/label-nodes.sh @@ -0,0 +1,32 @@ +#!/bin/sh +retry_kubectl() { + cmd=$1 + retries=0 + while [ $retries -lt 5 ]; do + $cmd + if [ $? -eq 0 ]; then + break + fi + retries=$((retries+1)) + sleep 5s + done + if [ $retries -eq 5 ]; then + echo "Error in executing $cmd" + exit 1 + fi +} + +for node in $(kubectl get nodes -o name); +do + echo "Current : $node" + node_name="${node##*/}" + echo "Apply label to the node" + kubectl label node $node_name connectivity-test=true + kubectl label node $node_name scale-test=true + if [ $? -eq 0 ]; then + echo "Label applied to the node" + else + echo "Error in applying label to the node $node_name" + fi + sleep 2s +done diff --git a/test/scale/templates/real-nginx-deployment.yaml b/test/scale/templates/real-nginx-deployment.yaml new file mode 100644 index 0000000000..0bdd413f92 --- /dev/null +++ b/test/scale/templates/real-nginx-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: TEMP_NAME + namespace: scale-test + labels: + app: scale-test + is-real: "true" + name: TEMP_NAME +spec: + replicas: TEMP_REPLICAS + selector: + matchLabels: + app: scale-test + is-real: "true"OTHER_LABELS_6_SPACES + template: + metadata: + labels: + app: scale-test + name: TEMP_NAME + is-real: "true"OTHER_LABELS_8_SPACES + spec: + nodeSelector: + scale-test: "true" + containers: + - name: nginx + image: nginx:1 + ports: + - name: http + containerPort: 80 diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh index 433264cffb..ecab516438 100755 --- a/test/scale/test-scale.sh +++ b/test/scale/test-scale.sh @@ -41,6 +41,7 @@ OPTIONAL PARAMETERS: --debug-exit-after-print-counts skip scale test. Just print out counts of things to be created and counts of IPSets/ACLs that NPM would create --num-real-services cluster ip service for the real deployments scheduled. Each svc will point to the respective deployment(having pods) Default is 0 --debug-exit-after-generation skip scale test. Exit after generating templates + --num-real-nginx-deployments create real-nginx-deployments, real-deployments should be set to 0 when this is in use. Default is 0 OPTIONAL PARAMETERS TO TEST DELETION: --sleep-after-creation= seconds to sleep after creating everything. Default is 0 @@ -79,6 +80,9 @@ while [[ $# -gt 0 ]]; do --num-real-deployments=*) numRealDeployments="${1#*=}" ;; + --num-real-nginx-deployments=*) + numRealNginxDeployments="${1#*=}" + ;; --num-real-replicas=*) numRealReplicas="${1#*=}" ;; @@ -181,6 +185,7 @@ if [[ -z $KUBECTL ]]; then KUBECTL="kubectl" fi if [[ -z $numRealServices ]]; then numRealServices=0; fi +if [[ -z $numRealNginxDeployments ]]; then numRealNginxDeployments=0; fi if [[ -z $deletePodsInterval ]]; then deletePodsInterval=60; fi if [[ -z $deletePodsTimes ]]; then deletePodsTimes=1; fi if [[ -z $deleteLabelsInterval ]]; then deleteLabelsInterval=60; fi @@ -213,7 +218,7 @@ if [[ $numRealPods -gt 0 ]]; then extraIPSets=$(( $extraIPSets + 2 )) fi if [[ $numRealServices -gt 0 ]]; then - extraIPSets=$(( $extraIPSets + $numRealDeployments )) + extraIPSets=$(( $extraIPSets + $numRealDeployments + $numRealNginxDeployments)) fi numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment + $extraIPSets )) # 3 basic members are [all-ns,kubernetes.io/metadata.name,kubernetes.io/metadata.name:scale-test] @@ -230,6 +235,7 @@ numKwokDeployments=$numKwokDeployments numKwokReplicas=$numKwokReplicas numRealServices=$numRealServices numRealDeployments=$numRealDeployments +numRealNginxDeployments=$numRealNginxDeployments numRealReplicas=$numRealReplicas numSharedLabelsPerPod=$numSharedLabelsPerPod numUniqueLabelsPerPod=$numUniqueLabelsPerPod @@ -293,13 +299,21 @@ mkdir -p generated/networkpolicies/applied mkdir -p generated/networkpolicies/unapplied mkdir -p generated/kwok-nodes mkdir -p generated/deployments/real/ +mkdir -p generated/deployments/real-nginx/ mkdir -p generated/deployments/kwok/ mkdir -p generated/services/real/ generateDeployments() { - local numDeployments=$1 + if [[ $numRealNginxDeployments -gt 0 ]]; then + local numDeployments=$numRealNginxDeployments + local depKind="real-nginx" + else + local numDeployments=$1 + local depKind=$3 + fi + local numReplicas=$2 - local depKind=$3 + for i in $(seq -f "%05g" 1 $numDeployments); do name="$depKind-dep-$i" @@ -338,7 +352,11 @@ generateServices() { outFile=generated/services/$serviceKind/$name.yaml sed "s/TEMP_NAME/$name/g" templates/$serviceKind-service.yaml > $outFile - sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-dep-$i/g" $outFile + if [[ $numRealNginxDeployments -gt 0 ]]; then + sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-nginx-dep-$i/g" $outFile + else + sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-dep-$i/g" $outFile + fi done } @@ -441,6 +459,9 @@ fi if [[ $numRealPods -gt 0 ]]; then $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/real/ fi +if [[ $numRealNginxDeployments -gt 0 ]]; then + $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/real-nginx/ +fi if [[ $numKwokPods -gt 0 ]]; then $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/kwok/ fi From 407c9f02a6c19dee6b2376e0f2f79fd58287f0b7 Mon Sep 17 00:00:00 2001 From: camrynl <31013536+camrynl@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:05:11 -0700 Subject: [PATCH 2/4] address comments and define --realPodType --- ...ment.yaml => real-agnhost-deployment.yaml} | 0 test/scale/test-scale.sh | 37 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) rename test/scale/templates/{real-deployment.yaml => real-agnhost-deployment.yaml} (100%) diff --git a/test/scale/templates/real-deployment.yaml b/test/scale/templates/real-agnhost-deployment.yaml similarity index 100% rename from test/scale/templates/real-deployment.yaml rename to test/scale/templates/real-agnhost-deployment.yaml diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh index ecab516438..d863a1bd59 100755 --- a/test/scale/test-scale.sh +++ b/test/scale/test-scale.sh @@ -41,7 +41,7 @@ OPTIONAL PARAMETERS: --debug-exit-after-print-counts skip scale test. Just print out counts of things to be created and counts of IPSets/ACLs that NPM would create --num-real-services cluster ip service for the real deployments scheduled. Each svc will point to the respective deployment(having pods) Default is 0 --debug-exit-after-generation skip scale test. Exit after generating templates - --num-real-nginx-deployments create real-nginx-deployments, real-deployments should be set to 0 when this is in use. Default is 0 + --real-pod-type select deployment type. Options are agnhost or nginx. Default is agnhost OPTIONAL PARAMETERS TO TEST DELETION: --sleep-after-creation= seconds to sleep after creating everything. Default is 0 @@ -83,6 +83,9 @@ while [[ $# -gt 0 ]]; do --num-real-nginx-deployments=*) numRealNginxDeployments="${1#*=}" ;; + --real-pod-type=*) + realPodType="${1#*=}" + ;; --num-real-replicas=*) numRealReplicas="${1#*=}" ;; @@ -184,6 +187,9 @@ fi if [[ -z $KUBECTL ]]; then KUBECTL="kubectl" fi +if [[ -z $realPodType ]]; then + realPodType="agnhost" +fi if [[ -z $numRealServices ]]; then numRealServices=0; fi if [[ -z $numRealNginxDeployments ]]; then numRealNginxDeployments=0; fi if [[ -z $deletePodsInterval ]]; then deletePodsInterval=60; fi @@ -298,21 +304,14 @@ test -d generated && rm -rf generated/ mkdir -p generated/networkpolicies/applied mkdir -p generated/networkpolicies/unapplied mkdir -p generated/kwok-nodes -mkdir -p generated/deployments/real/ -mkdir -p generated/deployments/real-nginx/ +mkdir -p generated/deployments/real-$realPodType/ mkdir -p generated/deployments/kwok/ mkdir -p generated/services/real/ generateDeployments() { - if [[ $numRealNginxDeployments -gt 0 ]]; then - local numDeployments=$numRealNginxDeployments - local depKind="real-nginx" - else - local numDeployments=$1 - local depKind=$3 - fi - + local numDeployments=$1 local numReplicas=$2 + local depKind=$3 for i in $(seq -f "%05g" 1 $numDeployments); do @@ -346,25 +345,22 @@ generateServices() { local numServices=$1 local numDeployments=$2 local serviceKind=$3 + local depKind=$4 for i in $(seq -f "%05g" 1 $numServices); do name="$serviceKind-svc-$i" outFile=generated/services/$serviceKind/$name.yaml sed "s/TEMP_NAME/$name/g" templates/$serviceKind-service.yaml > $outFile - if [[ $numRealNginxDeployments -gt 0 ]]; then - sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-nginx-dep-$i/g" $outFile - else - sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-dep-$i/g" $outFile - fi + sed -i "s/TEMP_DEPLOYMENT_NAME/$serviceKind-$depKind-dep-$i/g" $outFile done } echo "Generating yamls..." generateDeployments $numKwokDeployments $numKwokReplicas kwok -generateDeployments $numRealDeployments $numRealReplicas real -generateServices $numRealServices $numRealDeployments real +generateDeployments $numRealDeployments $numRealReplicas real-$realPodType +generateServices $numRealServices $numRealDeployments real $realPodType for j in $(seq 1 $numNetworkPolicies); do valNum=$j @@ -457,10 +453,7 @@ if [[ $numKwokNodes -gt 0 ]]; then $KUBECTL $KUBECONFIG_ARG apply -f generated/kwok-nodes/ fi if [[ $numRealPods -gt 0 ]]; then - $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/real/ -fi -if [[ $numRealNginxDeployments -gt 0 ]]; then - $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/real-nginx/ + $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/real-$realPodType/ fi if [[ $numKwokPods -gt 0 ]]; then $KUBECTL $KUBECONFIG_ARG apply -f generated/deployments/kwok/ From 119e2f1b167fe00b381eabb963dac8fa5e1a98d1 Mon Sep 17 00:00:00 2001 From: camrynl <31013536+camrynl@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:06:50 -0700 Subject: [PATCH 3/4] remove instances of numRealNginxDeployment --- test/scale/test-scale.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/scale/test-scale.sh b/test/scale/test-scale.sh index d863a1bd59..1a24f7fe7c 100755 --- a/test/scale/test-scale.sh +++ b/test/scale/test-scale.sh @@ -80,9 +80,6 @@ while [[ $# -gt 0 ]]; do --num-real-deployments=*) numRealDeployments="${1#*=}" ;; - --num-real-nginx-deployments=*) - numRealNginxDeployments="${1#*=}" - ;; --real-pod-type=*) realPodType="${1#*=}" ;; @@ -191,7 +188,6 @@ if [[ -z $realPodType ]]; then realPodType="agnhost" fi if [[ -z $numRealServices ]]; then numRealServices=0; fi -if [[ -z $numRealNginxDeployments ]]; then numRealNginxDeployments=0; fi if [[ -z $deletePodsInterval ]]; then deletePodsInterval=60; fi if [[ -z $deletePodsTimes ]]; then deletePodsTimes=1; fi if [[ -z $deleteLabelsInterval ]]; then deleteLabelsInterval=60; fi @@ -224,7 +220,7 @@ if [[ $numRealPods -gt 0 ]]; then extraIPSets=$(( $extraIPSets + 2 )) fi if [[ $numRealServices -gt 0 ]]; then - extraIPSets=$(( $extraIPSets + $numRealDeployments + $numRealNginxDeployments)) + extraIPSets=$(( $extraIPSets + $numRealDeployments)) fi numIPSetsAddedByNPM=$(( 4 + 2*$numTotalPods*$numUniqueLabelsPerPod + 2*$numSharedLabelsPerPod + 2*($numKwokDeployments+$numRealDeployments)*$numUniqueLabelsPerDeployment + $extraIPSets )) # 3 basic members are [all-ns,kubernetes.io/metadata.name,kubernetes.io/metadata.name:scale-test] @@ -241,7 +237,6 @@ numKwokDeployments=$numKwokDeployments numKwokReplicas=$numKwokReplicas numRealServices=$numRealServices numRealDeployments=$numRealDeployments -numRealNginxDeployments=$numRealNginxDeployments numRealReplicas=$numRealReplicas numSharedLabelsPerPod=$numSharedLabelsPerPod numUniqueLabelsPerPod=$numUniqueLabelsPerPod From 4ff9551eb0ba6f92711162f563027ddd453e6f6f Mon Sep 17 00:00:00 2001 From: camrynl <31013536+camrynl@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:51:12 -0700 Subject: [PATCH 4/4] move label-nodes script --- test/scale/{templates => }/label-nodes.sh | 29 +++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) rename test/scale/{templates => }/label-nodes.sh (59%) mode change 100644 => 100755 diff --git a/test/scale/templates/label-nodes.sh b/test/scale/label-nodes.sh old mode 100644 new mode 100755 similarity index 59% rename from test/scale/templates/label-nodes.sh rename to test/scale/label-nodes.sh index 3bcc0623bc..ec500f677b --- a/test/scale/templates/label-nodes.sh +++ b/test/scale/label-nodes.sh @@ -1,20 +1,19 @@ #!/bin/sh -retry_kubectl() { - cmd=$1 - retries=0 - while [ $retries -lt 5 ]; do - $cmd - if [ $? -eq 0 ]; then - break - fi - retries=$((retries+1)) - sleep 5s - done - if [ $retries -eq 5 ]; then - echo "Error in executing $cmd" - exit 1 +cmd=$1 +retries=0 +while [ $retries -lt 5 ]; do + $cmd + if [ $? -eq 0 ]; then + break fi -} + retries=$((retries+1)) + sleep 5s +done + +if [ $retries -eq 5 ]; then + echo "Error in executing $cmd" + exit 1 +fi for node in $(kubectl get nodes -o name); do