Skip to content

Commit 2bc231e

Browse files
author
Kubernetes Submit Queue
authored
Merge pull request kubernetes#61143 from satyasm/ipam-perf-cloud-mock
Automatic merge from submit-queue (batch tested with PRs 61402, 61143, 61427, 60592). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Performance tests and fix for IPAM controller. Tests the four modes of allocations. Can be run using ./test-performance.sh under tests/integration/ipamperf directory. See ./test-performance.sh -h for supported flags. **What this PR does / why we need it**: **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: Please see the implementation notes comment block in cloud.go for core details of how the mocking works. README.md has details on how the tests can be run on the command line. **Release note**: ```release-note Performance test framework and basic tests for the IPAM controller, to simulate behavior of the four supported modes under lightly loaded and loaded conditions, where load is defined as the number of operations to perform as against the configured kubernetes API server QPS. ```
2 parents 66ecee9 + 77da96c commit 2bc231e

File tree

12 files changed

+925
-0
lines changed

12 files changed

+925
-0
lines changed

pkg/cloudprovider/providers/gce/support.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,14 @@ func (l *gceRateLimiter) Accept(ctx context.Context, key *cloud.RateLimitKey) er
6464
}
6565
return nil
6666
}
67+
68+
// CreateGCECloudWithCloud is a helper function to create an instance of GCECloud with the
69+
// given Cloud interface implementation. Typical usage is to use cloud.NewMockGCE to get a
70+
// handle to a mock Cloud instance and then use that for testing.
71+
func CreateGCECloudWithCloud(config *CloudConfig, c cloud.Cloud) (*GCECloud, error) {
72+
gceCloud, err := CreateGCECloud(config)
73+
if err == nil {
74+
gceCloud.c = c
75+
}
76+
return gceCloud, err
77+
}

test/integration/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ filegroup(
4848
"//test/integration/examples:all-srcs",
4949
"//test/integration/framework:all-srcs",
5050
"//test/integration/garbagecollector:all-srcs",
51+
"//test/integration/ipamperf:all-srcs",
5152
"//test/integration/master:all-srcs",
5253
"//test/integration/metrics:all-srcs",
5354
"//test/integration/objectmeta:all-srcs",

test/integration/ipamperf/BUILD

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package(default_visibility = ["//visibility:public"])
2+
3+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
4+
5+
go_test(
6+
name = "go_default_test",
7+
size = "large",
8+
srcs = [
9+
"ipam_test.go",
10+
"main_test.go",
11+
],
12+
embed = [":go_default_library"],
13+
tags = ["integration"],
14+
deps = [
15+
"//pkg/api/testapi:go_default_library",
16+
"//pkg/controller/nodeipam:go_default_library",
17+
"//pkg/controller/nodeipam/ipam:go_default_library",
18+
"//test/integration/framework:go_default_library",
19+
"//test/integration/util:go_default_library",
20+
"//vendor/github.com/golang/glog:go_default_library",
21+
"//vendor/k8s.io/api/core/v1:go_default_library",
22+
"//vendor/k8s.io/client-go/informers:go_default_library",
23+
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
24+
"//vendor/k8s.io/client-go/rest:go_default_library",
25+
],
26+
)
27+
28+
filegroup(
29+
name = "package-srcs",
30+
srcs = glob(["**"]),
31+
tags = ["automanaged"],
32+
visibility = ["//visibility:private"],
33+
)
34+
35+
filegroup(
36+
name = "all-srcs",
37+
srcs = [":package-srcs"],
38+
tags = ["automanaged"],
39+
visibility = ["//visibility:public"],
40+
)
41+
42+
go_library(
43+
name = "go_default_library",
44+
srcs = [
45+
"cloud.go",
46+
"results.go",
47+
"util.go",
48+
],
49+
importpath = "k8s.io/kubernetes/test/integration/ipamperf",
50+
deps = [
51+
"//pkg/api/testapi:go_default_library",
52+
"//pkg/cloudprovider:go_default_library",
53+
"//pkg/cloudprovider/providers/gce/cloud:go_default_library",
54+
"//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library",
55+
"//pkg/controller/nodeipam/ipam:go_default_library",
56+
"//pkg/controller/nodeipam/ipam/cidrset:go_default_library",
57+
"//pkg/controller/util/node:go_default_library",
58+
"//test/integration/util:go_default_library",
59+
"//vendor/github.com/golang/glog:go_default_library",
60+
"//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
61+
"//vendor/google.golang.org/api/compute/v1:go_default_library",
62+
"//vendor/k8s.io/api/core/v1:go_default_library",
63+
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
64+
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
65+
"//vendor/k8s.io/client-go/informers:go_default_library",
66+
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
67+
"//vendor/k8s.io/client-go/rest:go_default_library",
68+
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
69+
],
70+
)

test/integration/ipamperf/README.md

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
IPAM Performance Test
2+
=====
3+
4+
Motivation
5+
-----
6+
We wanted to be able to test the behavior of the IPAM controller's under various scenarios,
7+
by mocking and monitoring the edges that the controller interacts with. This has the following goals:
8+
9+
- Save time on testing
10+
- To simulate various behaviors cheaply
11+
- To observe and model the ideal behavior of the IPAM controller code
12+
13+
Currently the test runs through the 4 different IPAM controller modes for cases where the kube API QPS is a)
14+
equal to and b) significantly less than the number of nodes being added to observe and quantify behavior.
15+
16+
How to run
17+
-------
18+
19+
```shell
20+
# In kubernetes root path
21+
make generated_files
22+
23+
cd test/integration/ipamperf
24+
./test-performance.sh
25+
```
26+
27+
The runner scripts support a few different options:
28+
29+
```shell
30+
./test-performance.sh -h
31+
usage: ./test-performance.sh [-h] [-d] [-r <pattern>] [-o <filename>]
32+
-h display this help message
33+
-d enable debug logs in tests
34+
-r <pattern> regex pattern to match for tests
35+
-o <filename> file to write JSON formatted results to
36+
```
37+
38+
The tests follow the pattern TestPerformance/{AllocatorType}-KubeQPS{X}-Nodes{Y}, where AllocatorType
39+
is one of
40+
41+
- RangeAllocator
42+
- IPAMFromCluster
43+
- CloudAllocator
44+
- IPAMFromCloud
45+
46+
and X represents the QPS configured for the kubernetes API client, and Y is the number of nodes to create.
47+
48+
The -d flags set the -v level for glog to 6, enabling nearly all of the debug logs in the code.
49+
50+
So to run the test for CloudAllocator with 10 nodes, one can run
51+
52+
```shell
53+
./test-performance.sh -r /CloudAllocator.*Nodes10$
54+
```
55+
56+
At the end of the test, a JSON format of the results for all the tests run is printed. Passing the -o option
57+
allows for also saving this JSON to a named file.
58+
59+
Code Organization
60+
-----
61+
The core of the tests are defined in [ipam_test.go](ipam_test.go), using the t.Run() helper to control parallelism
62+
as we want to able to start the master once. [cloud.go](cloud.go) contains the mock of the cloud server endpoint
63+
and can be configured to behave differently as needed by the various modes. The tracking of the node behavior and
64+
creation of the test results data is in [results.go](results.go).

test/integration/ipamperf/cloud.go

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package ipamperf
18+
19+
import (
20+
"context"
21+
"net"
22+
"sync"
23+
24+
beta "google.golang.org/api/compute/v0.beta"
25+
ga "google.golang.org/api/compute/v1"
26+
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
27+
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
28+
"k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset"
29+
"k8s.io/kubernetes/test/integration/util"
30+
)
31+
32+
// implemntation note:
33+
// ------------------
34+
// cloud.go implements hooks and handler functions for the MockGCE cloud in order to meet expectations
35+
// of cloud behavior from the IPAM controllers. The key constraint is that the IPAM code is spread
36+
// across both GA and Beta instances, which are distinct objects in the mock. We need to solve for
37+
//
38+
// 1. When a GET is called on an instance, we lazy create the instance with or without an assigned
39+
// ip alias as needed by the IPAM controller type
40+
// 2. When we assign an IP alias for an instance, both the GA and Beta instance have to agree on the
41+
// assigned alias range
42+
//
43+
// We solve both the problems by using a baseInstanceList which maintains a list of known instances,
44+
// and their pre-assigned ip-alias ranges (if needed). We then create GetHook for GA and Beta GetInstance
45+
// calls as closures over this betaInstanceList that can lookup base instance data.
46+
//
47+
// This has the advantage that once the Get hook pouplates the GCEMock with the base data, we then let the
48+
// rest of the mock code run as is.
49+
50+
// baseInstance tracks basic instance data needed by the IPAM controllers
51+
type baseInstance struct {
52+
name string
53+
zone string
54+
aliasRange string
55+
}
56+
57+
// baseInstanceList tracks a set of base instances
58+
type baseInstanceList struct {
59+
allocateCIDR bool
60+
clusterCIDR *net.IPNet
61+
subnetMaskSize int
62+
cidrSet *cidrset.CidrSet
63+
64+
lock sync.Mutex // protect access to instances
65+
instances map[meta.Key]*baseInstance
66+
}
67+
68+
// toGA is an utility method to return the baseInstance data as a GA Instance object
69+
func (bi *baseInstance) toGA() *ga.Instance {
70+
inst := &ga.Instance{Name: bi.name, Zone: bi.zone, NetworkInterfaces: []*ga.NetworkInterface{{}}}
71+
if bi.aliasRange != "" {
72+
inst.NetworkInterfaces[0].AliasIpRanges = []*ga.AliasIpRange{
73+
{IpCidrRange: bi.aliasRange, SubnetworkRangeName: util.TestSecondaryRangeName},
74+
}
75+
}
76+
return inst
77+
}
78+
79+
// toGA is an utility method to return the baseInstance data as a beta Instance object
80+
func (bi *baseInstance) toBeta() *beta.Instance {
81+
inst := &beta.Instance{Name: bi.name, Zone: bi.zone, NetworkInterfaces: []*beta.NetworkInterface{{}}}
82+
if bi.aliasRange != "" {
83+
inst.NetworkInterfaces[0].AliasIpRanges = []*beta.AliasIpRange{
84+
{IpCidrRange: bi.aliasRange, SubnetworkRangeName: util.TestSecondaryRangeName},
85+
}
86+
}
87+
return inst
88+
}
89+
90+
// newBaseInstanceList is the baseInstanceList constructor
91+
func newBaseInstanceList(allocateCIDR bool, clusterCIDR *net.IPNet, subnetMaskSize int) *baseInstanceList {
92+
cidrSet, _ := cidrset.NewCIDRSet(clusterCIDR, subnetMaskSize)
93+
return &baseInstanceList{
94+
allocateCIDR: allocateCIDR,
95+
clusterCIDR: clusterCIDR,
96+
subnetMaskSize: subnetMaskSize,
97+
cidrSet: cidrSet,
98+
instances: make(map[meta.Key]*baseInstance),
99+
}
100+
}
101+
102+
// getOrCreateBaseInstance lazily creates a new base instance, assigning if allocateCIDR is true
103+
func (bil *baseInstanceList) getOrCreateBaseInstance(key *meta.Key) *baseInstance {
104+
bil.lock.Lock()
105+
defer bil.lock.Unlock()
106+
107+
inst, found := bil.instances[*key]
108+
if !found {
109+
inst = &baseInstance{name: key.Name, zone: key.Zone}
110+
if bil.allocateCIDR {
111+
nextRange, _ := bil.cidrSet.AllocateNext()
112+
inst.aliasRange = nextRange.String()
113+
}
114+
bil.instances[*key] = inst
115+
}
116+
return inst
117+
}
118+
119+
// newGAGetHook creates a new closure with the current baseInstanceList to be used as a MockInstances.GetHook
120+
func (bil *baseInstanceList) newGAGetHook() func(ctx context.Context, key *meta.Key, m *cloud.MockInstances) (bool, *ga.Instance, error) {
121+
return func(ctx context.Context, key *meta.Key, m *cloud.MockInstances) (bool, *ga.Instance, error) {
122+
m.Lock.Lock()
123+
defer m.Lock.Unlock()
124+
125+
if _, found := m.Objects[*key]; !found {
126+
m.Objects[*key] = &cloud.MockInstancesObj{Obj: bil.getOrCreateBaseInstance(key).toGA()}
127+
}
128+
return false, nil, nil
129+
}
130+
}
131+
132+
// newBetaGetHook creates a new closure with the current baseInstanceList to be used as a MockBetaInstances.GetHook
133+
func (bil *baseInstanceList) newBetaGetHook() func(ctx context.Context, key *meta.Key, m *cloud.MockBetaInstances) (bool, *beta.Instance, error) {
134+
return func(ctx context.Context, key *meta.Key, m *cloud.MockBetaInstances) (bool, *beta.Instance, error) {
135+
m.Lock.Lock()
136+
defer m.Lock.Unlock()
137+
138+
if _, found := m.Objects[*key]; !found {
139+
m.Objects[*key] = &cloud.MockInstancesObj{Obj: bil.getOrCreateBaseInstance(key).toBeta()}
140+
}
141+
return false, nil, nil
142+
}
143+
}
144+
145+
// newMockCloud returns a mock GCE instance with the appropriate handlers hooks
146+
func (bil *baseInstanceList) newMockCloud() cloud.Cloud {
147+
c := cloud.NewMockGCE(nil)
148+
149+
// insert hooks to lazy create a instance when needed
150+
c.MockInstances.GetHook = bil.newGAGetHook()
151+
c.MockBetaInstances.GetHook = bil.newBetaGetHook()
152+
153+
return c
154+
}

0 commit comments

Comments
 (0)