Merge pull request #1822 from mborsz/miti

Mitigate kubernetes/kubernetes#96635 in load test
kubernetes · Jun 7, 2021 · d15b55a · d15b55a
2 parents fbadf20 + ad71359
commit d15b55a
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 12 deletions.
diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -84,9 +84,17 @@ tuningSets:
   parallelismLimitedLoad:
     parallelismLimit: {{$schedulerThroughputNamespaces}}
 # TODO(https://github.com/kubernetes/perf-tests/issues/1024): This TuningSet is used only for pod-startup-latency, get rid of it
+# Uniform5qps: for each running phase, use 5 qps.
 - name: Uniform5qps
   qpsLoad:
     qps: 5
+# Global100qps: use 100 qps globally:
+# * split equally qps among all running phases
+# * if some phase finishes, other phases will get more qps.
+- name: Global100qps
+  globalQPSLoad:
+    qps: 100
+    burst: 1
 - name: RandomizedSaturationTimeLimited
   RandomizedTimeLimitedLoad:
     timeLimit: {{$saturationTime}}s
@@ -130,6 +138,18 @@ steps:
       - basename: daemonset-priorityclass
         objectTemplatePath: daemonset-priorityclass.yaml
 
+# Moved from reconcile-objects.yaml to mitigate https://github.com/kubernetes/kubernetes/issues/96635.
+# TODO(https://github.com/kubernetes/perf-tests/issues/1823): Merge back to reconcile-objects.yaml once the k/k bug is fixed.
+- module:
+    path: /modules/configmaps-secrets.yaml
+    params:
+      actionName: create
+      tuningSet: Global100qps
+      namespaces: {{$namespaces}}
+      bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
+      mediumDeploymentsPerNamespace: {{$mediumDeploymentsPerNamespace}}
+      smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespace}}
+
 - module:
     path: /modules/reconcile-objects.yaml
     params:
@@ -306,6 +326,16 @@ steps:
       pvSmallStatefulSetSize: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}}
       pvMediumStatefulSetSize: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}}
 
+- module:
+    path: /modules/configmaps-secrets.yaml
+    params:
+      actionName: delete
+      tuningSet: Global100qps
+      namespaces: {{$namespaces}}
+      bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
+      mediumDeploymentsPerNamespace: {{$mediumDeploymentsPerNamespace}}
+      smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespace}}
+
 - name: Deleting PriorityClass for DaemonSets
   phases:
     - replicasPerNamespace: 0

diff --git a/clusterloader2/testing/load/modules/configmaps-secrets.yaml b/clusterloader2/testing/load/modules/configmaps-secrets.yaml
@@ -0,0 +1,45 @@
+# To mitigate https://github.com/kubernetes/kubernetes/issues/96635 we have to delete
+# configmaps and secrets only AFTER the pods using that configmap/secret are deleted.
+# TODO(mborsz): Merge this back to reconcile-objects.yaml once the issue above is fixed.
+
+{{$actionName := printf "%s objects" .actionName}}
+{{$namespaces := .namespaces}}
+{{$tuningSet := .tuningSet}}
+
+{{$bigDeploymentsPerNamespace := .bigDeploymentsPerNamespace}}
+{{$mediumDeploymentsPerNamespace := .mediumDeploymentsPerNamespace}}
+{{$smallDeploymentsPerNamespace := .smallDeploymentsPerNamespace}}
+
+steps:
+- name: {{$actionName}} configmaps and secrets
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$bigDeploymentsPerNamespace}}
+    tuningSet: {{$tuningSet}}
+    objectBundle:
+    - basename: big-deployment
+      objectTemplatePath: configmap.yaml
+    - basename: big-deployment
+      objectTemplatePath: secret.yaml
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$mediumDeploymentsPerNamespace}}
+    tuningSet: {{$tuningSet}}
+    objectBundle:
+    - basename: medium-deployment
+      objectTemplatePath: configmap.yaml
+    - basename: medium-deployment
+      objectTemplatePath: secret.yaml
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$smallDeploymentsPerNamespace}}
+    tuningSet: {{$tuningSet}}
+    objectBundle:
+    - basename: small-deployment
+      objectTemplatePath: configmap.yaml
+    - basename: small-deployment
+      objectTemplatePath: secret.yaml
diff --git a/clusterloader2/testing/load/modules/reconcile-objects.yaml b/clusterloader2/testing/load/modules/reconcile-objects.yaml
@@ -96,10 +96,6 @@ steps:
     replicasPerNamespace: {{$bigDeploymentsPerNamespace}}
     tuningSet: {{$tuningSet}}
     objectBundle:
-    - basename: big-deployment
-      objectTemplatePath: configmap.yaml
-    - basename: big-deployment
-      objectTemplatePath: secret.yaml
 {{if $ENABLE_NETWORKPOLICIES}}
     - basename: big-deployment
       objectTemplatePath: networkpolicy.yaml
@@ -119,10 +115,6 @@ steps:
     replicasPerNamespace: {{$mediumDeploymentsPerNamespace}}
     tuningSet: {{$tuningSet}}
     objectBundle:
-    - basename: medium-deployment
-      objectTemplatePath: configmap.yaml
-    - basename: medium-deployment
-      objectTemplatePath: secret.yaml
 {{if $ENABLE_NETWORKPOLICIES}}
     - basename: medium-deployment
       objectTemplatePath: networkpolicy.yaml
@@ -141,10 +133,6 @@ steps:
     replicasPerNamespace: {{$smallDeploymentsPerNamespace}}
     tuningSet: {{$tuningSet}}
     objectBundle:
-    - basename: small-deployment
-      objectTemplatePath: configmap.yaml
-    - basename: small-deployment
-      objectTemplatePath: secret.yaml
 {{if $ENABLE_NETWORKPOLICIES}}
     - basename: small-deployment
       objectTemplatePath: networkpolicy.yaml