Fix inter-pod affinity scheduler benchmarks

kubernetes · Dec 10, 2019 · a051c59 · a051c59
1 parent 50437b4
commit a051c59
Showing 1 changed file with 62 additions and 31 deletions.
diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go
@@ -45,20 +45,23 @@ var (
 	// From PV controller
 	annBindCompleted = "pv.kubernetes.io/bind-completed"
 
-	tests = []struct{ nodes, existingPods, minPods int }{
+	defaultTests = []struct{ nodes, existingPods, minPods int }{
 		{nodes: 500, existingPods: 500, minPods: 1000},
 		{nodes: 5000, existingPods: 5000, minPods: 1000},
 	}
+	testNamespace  = "sched-test"
+	setupNamespace = "sched-setup"
 )
 
 // BenchmarkScheduling benchmarks the scheduling rate when the cluster has
 // various quantities of nodes and scheduled pods.
 func BenchmarkScheduling(b *testing.B) {
 	testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -67,15 +70,26 @@ func BenchmarkScheduling(b *testing.B) {
 // PodAntiAffinity rules when the cluster has various quantities of nodes and
 // scheduled pods.
 func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
+	// Since the pods has anti affinity to each other, the number of pods to schedule
+	// can't exceed the number of nodes (the topology used in the test)
+	tests := []struct{ nodes, existingPods, minPods int }{
+		{nodes: 500, existingPods: 100, minPods: 400},
+		{nodes: 5000, existingPods: 1000, minPods: 1000},
+	}
 	testBasePod := makeBasePodWithPodAntiAffinity(
 		map[string]string{"name": "test", "color": "green"},
 		map[string]string{"color": "green"})
-	// The test strategy creates pods with anti-affinity for each other.
+	// The test strategy creates pods with anti-affinity to each other, each pod ending up in a separate node.
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
 	for _, test := range tests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
+			var nodeStrategies []testutils.CountToStrategy
+			for i := 0; i < test.nodes; i++ {
+				nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
+				nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
+			}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -88,10 +102,11 @@ func BenchmarkSchedulingSecrets(b *testing.B) {
 	// The test strategy creates pods with a secret.
 	testBasePod := makeBasePodWithSecret()
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -104,10 +119,11 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) {
 	baseClaim := makeBasePersistentVolumeClaim()
 	basePod := makeBasePod()
 	testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod)
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -134,12 +150,13 @@ func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) {
 		},
 	}
 	nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{csilibplugins.AWSEBSInTreePluginName})
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
 			defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)()
 			defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)()
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -164,10 +181,11 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) {
 		},
 	}
 	nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{})
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -183,10 +201,11 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) {
 	// The test strategy creates pods with affinity for each other.
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
 	nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -201,11 +220,15 @@ func BenchmarkSchedulingPreferredPodAffinity(b *testing.B) {
 	)
 	// The test strategy creates pods with affinity for each other.
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
-	nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			var nodeStrategies []testutils.CountToStrategy
+			for i := 0; i < test.nodes; i++ {
+				nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
+				nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
+			}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -218,13 +241,17 @@ func BenchmarkSchedulingPreferredPodAntiAffinity(b *testing.B) {
 		map[string]string{"foo": ""},
 		map[string]string{"foo": ""},
 	)
-	// The test strategy creates pods with affinity for each other.
+	// The test strategy creates pods with anti affinity to each other.
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
-	nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			var nodeStrategies []testutils.CountToStrategy
+			for i := 0; i < test.nodes; i++ {
+				nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
+				nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
+			}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -237,10 +264,11 @@ func BenchmarkSchedulingNodeAffinity(b *testing.B) {
 	// The test strategy creates pods with node-affinity for each other.
 	testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
 	nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
-	for _, test := range tests {
+	for _, test := range defaultTests {
 		name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
 		b.Run(name, func(b *testing.B) {
-			benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
+			nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
+			benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
 		})
 	}
 }
@@ -263,6 +291,7 @@ func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string)
 						MatchLabels: affinityLabels,
 					},
 					TopologyKey: v1.LabelHostname,
+					Namespaces:  []string{testNamespace, setupNamespace},
 				},
 			},
 		},
@@ -289,6 +318,7 @@ func makeBasePodWithPreferredPodAntiAffinity(podLabels, affinityLabels map[strin
 							MatchLabels: affinityLabels,
 						},
 						TopologyKey: v1.LabelHostname,
+						Namespaces:  []string{testNamespace, setupNamespace},
 					},
 					Weight: 1,
 				},
@@ -317,6 +347,7 @@ func makeBasePodWithPreferredPodAffinity(podLabels, affinityLabels map[string]st
 							MatchLabels: affinityLabels,
 						},
 						TopologyKey: v1.LabelHostname,
+						Namespaces:  []string{testNamespace, setupNamespace},
 					},
 					Weight: 1,
 				},
@@ -344,6 +375,7 @@ func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string)
 						MatchLabels: affinityZoneLabels,
 					},
 					TopologyKey: v1.LabelZoneFailureDomain,
+					Namespaces:  []string{testNamespace, setupNamespace},
 				},
 			},
 		},
@@ -384,8 +416,8 @@ func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod {
 // and specific number of pods already scheduled.
 // This will schedule numExistingPods pods before the benchmark starts, and at
 // least minPods pods during the benchmark.
-func benchmarkScheduling(numNodes, numExistingPods, minPods int,
-	nodeStrategy testutils.PrepareNodeStrategy,
+func benchmarkScheduling(numExistingPods, minPods int,
+	nodeStrategies []testutils.CountToStrategy,
 	testPodStrategy testutils.TestPodCreateStrategy,
 	b *testing.B) {
 	if b.N < minPods {
@@ -396,16 +428,15 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
 
 	nodePreparer := framework.NewIntegrationTestNodePreparer(
 		clientset,
-		[]testutils.CountToStrategy{{Count: numNodes, Strategy: nodeStrategy}},
-		"scheduler-perf-",
-	)
+		nodeStrategies,
+		"scheduler-perf-")
 	if err := nodePreparer.PrepareNodes(); err != nil {
 		klog.Fatalf("%v", err)
 	}
 	defer nodePreparer.CleanupNodes()
 
 	config := testutils.NewTestPodCreatorConfig()
-	config.AddStrategy("sched-setup", numExistingPods, testPodStrategy)
+	config.AddStrategy(setupNamespace, numExistingPods, testPodStrategy)
 	podCreator := testutils.NewTestPodCreator(clientset, config)
 	podCreator.CreatePods()
 
@@ -439,7 +470,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
 	// start benchmark
 	b.ResetTimer()
 	config = testutils.NewTestPodCreatorConfig()
-	config.AddStrategy("sched-test", b.N, testPodStrategy)
+	config.AddStrategy(testNamespace, b.N, testPodStrategy)
 	podCreator = testutils.NewTestPodCreator(clientset, config)
 	podCreator.CreatePods()