Merge pull request #1998 from alculquicondor/job-metrics

Add job load test: create multiple jobs based on the number of nodes
kubernetes · Mar 17, 2022 · 4d7fdf3 · 4d7fdf3
2 parents 17c4884 + ebacb8d
commit 4d7fdf3
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 15 deletions.
diff --git a/clusterloader2/pkg/measurement/common/wait_for_jobs.go b/clusterloader2/pkg/measurement/common/wait_for_jobs.go
@@ -46,7 +46,7 @@ const (
 	defaultWaitForFinishedJobsTimeout = 10 * time.Minute
 	waitForFinishedJobsName           = "WaitForFinishedJobs"
 	waitForFinishedJobsWorkers        = 1
-	checkFinishedJobsInterval         = 5 * time.Second
+	checkFinishedJobsInterval         = time.Second
 )
 
 func init() {

diff --git a/clusterloader2/testing/batch/config.yaml b/clusterloader2/testing/batch/config.yaml
@@ -1,12 +1,31 @@
+{{$MODE := DefaultParam .MODE "Indexed"}}
+{{$NODES_PER_NAMESPACE := MinInt .Nodes (DefaultParam .NODES_PER_NAMESPACE 100)}}
+{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}}
+{{$LOAD_TEST_THROUGHPUT := DefaultParam .CL2_LOAD_TEST_THROUGHPUT 10}}
+
+{{$totalPods := MultiplyInt $PODS_PER_NODE .Nodes}}
+{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}}
+{{$podsPerNamespace := DivideInt $totalPods $namespaces}}
+
+# small_job: 1/2 of namespace pods should be in small Jobs.
+{{$smallJobSize := 5}}
+{{$smallJobsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 2 $smallJobSize)}}
+# medium_job: 1/4 of namespace pods should be in medium Jobs.
+{{$mediumJobSize := 20}}
+{{$mediumJobsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $mediumJobSize)}}
+# Large_job: 1/4 of namespace pods should be in large Jobs.
+{{$largeJobSize := 400}}
+{{$largeJobsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $largeJobSize)}}
+
 name: batch
 
 namespace:
-  number: 1
+  number: {{$namespaces}}
 
 tuningSets:
-- name: Uniform1qps
+- name: UniformQPS
   qpsLoad:
-    qps: 1
+    qps: {{$LOAD_TEST_THROUGHPUT}}
 
 steps:
 - name: Start measurements
@@ -16,22 +35,45 @@ steps:
     Params:
       action: start
       labelSelector: group = test-job
-- name: Create job
+- name: Create {{$MODE}} jobs
   phases:
   - namespaceRange:
       min: 1
-      max: 1
-    replicasPerNamespace: 1
-    tuningSet: Uniform1qps
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$smallJobsPerNamespace}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: small
+      objectTemplatePath: "job.yaml"
+      templateFillMap:
+        Replicas: {{$smallJobSize}}
+        Mode: {{$MODE}}
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$mediumJobsPerNamespace}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: medium
+      objectTemplatePath: "job.yaml"
+      templateFillMap:
+        Replicas: {{$mediumJobSize}}
+        Mode: {{$MODE}}
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$largeJobsPerNamespace}}
+    tuningSet: UniformQPS
     objectBundle:
-    - basename: test-job
+    - basename: large
       objectTemplatePath: "job.yaml"
       templateFillMap:
-        Replicas: 10
-- name: Wait for jobs to finish
+        Replicas: {{$largeJobSize}}
+        Mode: {{$MODE}}
+- name: Wait for {{$MODE}} jobs to finish
   measurements:
   - Identifier: WaitForFinishedJobs
     Method: WaitForFinishedJobs
     Params:
       action: gather
-      timeout: 1m
+      timeout: 10m
diff --git a/clusterloader2/testing/batch/job.yaml b/clusterloader2/testing/batch/job.yaml
@@ -7,13 +7,13 @@ metadata:
 spec:
   parallelism: {{.Replicas}}
   completions: {{.Replicas}}
+  completionMode: {{.Mode}}
   template:
     metadata:
       labels:
         group: test-pod
     spec:
       containers:
       - name: {{.Name}}
-        image: bash
-        args: ["-c", "exit"]
-      restartPolicy: Never
+        image: gcr.io/k8s-staging-perf-tests/sleep:v0.0.3
+      restartPolicy: Never