/
util.go
378 lines (315 loc) · 12.7 KB
/
util.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"context"
"fmt"
"os"
"runtime"
"strings"
"time"
"github.com/distribution/reference"
"github.com/google/uuid"
"github.com/kubernetes-sigs/cri-tools/pkg/common"
"gopkg.in/yaml.v3"
internalapi "k8s.io/cri-api/pkg/apis"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/kubernetes/pkg/kubelet/cri/remote"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var (
// the callbacks to run during BeforeSuite
beforeSuiteCallbacks []func()
// DefaultPodLabels are labels used by default in pods
DefaultPodLabels map[string]string
// DefaultContainerCommand is the default command used for containers
DefaultContainerCommand []string
// DefaultPauseCommand is the default command used for containers
DefaultPauseCommand []string
// DefaultLinuxPodLabels default pod labels for Linux
DefaultLinuxPodLabels = map[string]string{}
// DefaultLinuxContainerCommand default container command for Linux
DefaultLinuxContainerCommand = []string{"top"}
// DefaultLinuxPauseCommand default container command for Linux pause
DefaultLinuxPauseCommand = []string{"sh", "-c", "top"}
// DefaultLcowPodLabels default pod labels for Linux containers on Windows
DefaultLcowPodLabels = map[string]string{
"sandbox-platform": "linux/amd64",
}
// DefaultWindowsPodLabels default pod labels for Windows
DefaultWindowsPodLabels = map[string]string{}
// DefaultWindowsContainerCommand default container command for Windows
DefaultWindowsContainerCommand = []string{"cmd", "/c", "ping -t localhost"}
// DefaultWindowsPauseCommand default container pause command for Windows
DefaultWindowsPauseCommand = []string{"powershell", "-c", "ping -t localhost"}
)
const (
// DefaultUIDPrefix is a default UID prefix of PodSandbox
DefaultUIDPrefix string = "cri-test-uid"
// DefaultNamespacePrefix is a default namespace prefix of PodSandbox
DefaultNamespacePrefix string = "cri-test-namespace"
// DefaultAttempt is a default attempt prefix of PodSandbox or container
DefaultAttempt uint32 = 2
// DefaultStopContainerTimeout is the default timeout for stopping container
DefaultStopContainerTimeout int64 = 60
// DefaultLinuxContainerImage default container image for Linux
DefaultLinuxContainerImage string = DefaultRegistryE2ETestImagesPrefix + "busybox:1.29-2"
// DefaultWindowsContainerImage default container image for Windows
DefaultWindowsContainerImage string = DefaultLinuxContainerImage
)
// Set the constants based on operating system and flags
var _ = BeforeSuite(func() {
if runtime.GOOS != "windows" || TestContext.IsLcow {
DefaultPodLabels = DefaultLinuxPodLabels
DefaultContainerCommand = DefaultLinuxContainerCommand
DefaultPauseCommand = DefaultLinuxPauseCommand
TestContext.TestImageList.DefaultTestContainerImage = DefaultLinuxContainerImage
if TestContext.IsLcow {
DefaultPodLabels = DefaultLcowPodLabels
}
} else {
DefaultPodLabels = DefaultWindowsPodLabels
DefaultContainerCommand = DefaultWindowsContainerCommand
DefaultPauseCommand = DefaultWindowsPauseCommand
TestContext.TestImageList.DefaultTestContainerImage = DefaultWindowsContainerImage
}
// Load any custom image definitions:
err := TestContext.LoadYamlConfigFiles()
if err != nil {
panic(err)
}
for _, callback := range beforeSuiteCallbacks {
callback()
}
})
// AddBeforeSuiteCallback adds a callback to run during BeforeSuite
func AddBeforeSuiteCallback(callback func()) bool {
beforeSuiteCallbacks = append(beforeSuiteCallbacks, callback)
return true
}
// LoadCRIClient creates a InternalAPIClient.
func LoadCRIClient() (*InternalAPIClient, error) {
rService, err := remote.NewRemoteRuntimeService(
TestContext.RuntimeServiceAddr,
TestContext.RuntimeServiceTimeout,
nil,
)
if err != nil {
return nil, err
}
var imageServiceAddr = TestContext.ImageServiceAddr
if imageServiceAddr == "" {
// Fallback to runtime service endpoint
imageServiceAddr = TestContext.RuntimeServiceAddr
}
iService, err := remote.NewRemoteImageService(imageServiceAddr, TestContext.ImageServiceTimeout, nil)
if err != nil {
return nil, err
}
return &InternalAPIClient{
CRIRuntimeClient: rService,
CRIImageClient: iService,
}, nil
}
func nowStamp() string {
return time.Now().Format(time.StampMilli)
}
func log(level string, format string, args ...interface{}) {
fmt.Fprintf(GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
}
// Logf prints a info message.
func Logf(format string, args ...interface{}) {
log("INFO", format, args...)
}
// Failf prints an error message.
func Failf(format string, args ...interface{}) {
msg := fmt.Sprintf(format, args...)
log("INFO", msg)
Fail(nowStamp()+": "+msg, 1)
}
// ExpectNoError reports error if err is not nil.
func ExpectNoError(err error, explain ...interface{}) {
if err != nil {
Logf("Unexpected error occurred: %v", err)
}
ExpectWithOffset(1, err).NotTo(HaveOccurred(), explain...)
}
// NewUUID creates a new UUID string.
func NewUUID() string {
return uuid.New().String()
}
// RunDefaultPodSandbox runs a PodSandbox with default options.
func RunDefaultPodSandbox(c internalapi.RuntimeService, prefix string) string {
podSandboxName := prefix + NewUUID()
uid := DefaultUIDPrefix + NewUUID()
namespace := DefaultNamespacePrefix + NewUUID()
config := &runtimeapi.PodSandboxConfig{
Metadata: BuildPodSandboxMetadata(podSandboxName, uid, namespace, DefaultAttempt),
Linux: &runtimeapi.LinuxPodSandboxConfig{
CgroupParent: common.GetCgroupParent(context.TODO(), c),
},
Labels: DefaultPodLabels,
}
return RunPodSandbox(c, config)
}
// BuildPodSandboxMetadata builds PodSandboxMetadata.
func BuildPodSandboxMetadata(podSandboxName, uid, namespace string, attempt uint32) *runtimeapi.PodSandboxMetadata {
return &runtimeapi.PodSandboxMetadata{
Name: podSandboxName,
Uid: uid,
Namespace: namespace,
Attempt: attempt,
}
}
// RunPodSandbox runs a PodSandbox.
func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string {
podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler)
ExpectNoError(err, "failed to create PodSandbox: %v", err)
return podID
}
// RunPodSandboxError runs a PodSandbox and expects an error.
func RunPodSandboxError(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string {
podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler)
Expect(err).To(HaveOccurred())
return podID
}
// CreatePodSandboxForContainer creates a PodSandbox for creating containers.
func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtimeapi.PodSandboxConfig) {
podSandboxName := "create-PodSandbox-for-container-" + NewUUID()
uid := DefaultUIDPrefix + NewUUID()
namespace := DefaultNamespacePrefix + NewUUID()
config := &runtimeapi.PodSandboxConfig{
Metadata: BuildPodSandboxMetadata(podSandboxName, uid, namespace, DefaultAttempt),
Linux: &runtimeapi.LinuxPodSandboxConfig{
CgroupParent: common.GetCgroupParent(context.TODO(), c),
},
Labels: DefaultPodLabels,
}
podID := RunPodSandbox(c, config)
return podID, config
}
// BuildContainerMetadata builds containerMetadata.
func BuildContainerMetadata(containerName string, attempt uint32) *runtimeapi.ContainerMetadata {
return &runtimeapi.ContainerMetadata{
Name: containerName,
Attempt: attempt,
}
}
// CreateDefaultContainer creates a default container with default options.
func CreateDefaultContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) string {
containerName := prefix + NewUUID()
containerConfig := &runtimeapi.ContainerConfig{
Metadata: BuildContainerMetadata(containerName, DefaultAttempt),
Image: &runtimeapi.ImageSpec{Image: TestContext.TestImageList.DefaultTestContainerImage},
Command: DefaultContainerCommand,
Linux: &runtimeapi.LinuxContainerConfig{},
}
return CreateContainer(rc, ic, containerConfig, podID, podConfig)
}
// CreatePauseContainer creates a container with default pause options.
func CreatePauseContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) string {
containerName := prefix + NewUUID()
containerConfig := &runtimeapi.ContainerConfig{
Metadata: BuildContainerMetadata(containerName, DefaultAttempt),
Image: &runtimeapi.ImageSpec{Image: TestContext.TestImageList.DefaultTestContainerImage},
Command: DefaultPauseCommand,
Linux: &runtimeapi.LinuxContainerConfig{},
}
return CreateContainer(rc, ic, containerConfig, podID, podConfig)
}
// CreateContainerWithError creates a container but leave error check to caller
func CreateContainerWithError(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, config *runtimeapi.ContainerConfig, podID string, podConfig *runtimeapi.PodSandboxConfig) (string, error) {
// Pull the image if it does not exist.
imageName := config.Image.Image
if !strings.Contains(imageName, ":") {
imageName = imageName + ":latest"
Logf("Use latest as default image tag.")
}
status := ImageStatus(ic, imageName)
if status == nil {
PullPublicImage(ic, imageName, podConfig)
}
if config.Image.UserSpecifiedImage == "" {
config.Image.UserSpecifiedImage = imageName
}
By("Create container.")
containerID, err := rc.CreateContainer(context.TODO(), podID, config, podConfig)
return containerID, err
}
// CreateContainer creates a container with the prefix of containerName.
func CreateContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, config *runtimeapi.ContainerConfig, podID string, podConfig *runtimeapi.PodSandboxConfig) string {
containerID, err := CreateContainerWithError(rc, ic, config, podID, podConfig)
ExpectNoError(err, "failed to create container: %v", err)
Logf("Created container %q\n", containerID)
return containerID
}
// ImageStatus gets the status of the image named imageName.
func ImageStatus(c internalapi.ImageManagerService, imageName string) *runtimeapi.Image {
By("Get image status for image: " + imageName)
imageSpec := &runtimeapi.ImageSpec{
Image: imageName,
}
status, err := c.ImageStatus(context.TODO(), imageSpec, false)
ExpectNoError(err, "failed to get image status: %v", err)
return status.GetImage()
}
// ListImage list the image filtered by the image filter.
func ListImage(c internalapi.ImageManagerService, filter *runtimeapi.ImageFilter) []*runtimeapi.Image {
images, err := c.ListImages(context.TODO(), filter)
ExpectNoError(err, "Failed to get image list: %v", err)
return images
}
// PullPublicImage pulls the public image named imageName.
func PullPublicImage(c internalapi.ImageManagerService, imageName string, podConfig *runtimeapi.PodSandboxConfig) string {
ref, err := reference.ParseNamed(imageName)
if err == nil {
// Modify the image if it's a fully qualified image name
if TestContext.RegistryPrefix != DefaultRegistryPrefix {
r := fmt.Sprintf("%s/%s", TestContext.RegistryPrefix, reference.Path(ref))
ref, err = reference.ParseNamed(r)
ExpectNoError(err, "failed to parse new image name: %v", err)
}
imageName = ref.String()
if !strings.Contains(imageName, ":") {
imageName = imageName + ":latest"
Logf("Use latest as default image tag.")
}
} else if err == reference.ErrNameNotCanonical {
// Non canonical images can simply be prefixed
imageName = fmt.Sprintf("%s/%s", TestContext.RegistryPrefix, imageName)
} else {
Failf("Unable to parse imageName: %v", err)
}
By("Pull image : " + imageName)
imageSpec := &runtimeapi.ImageSpec{
Image: imageName,
}
id, err := c.PullImage(context.TODO(), imageSpec, nil, podConfig)
ExpectNoError(err, "failed to pull image: %v", err)
return id
}
// LoadYamlFile attempts to load the given YAML file into the given struct.
func LoadYamlFile(filepath string, obj interface{}) error {
Logf("Attempting to load YAML file %q into %+v", filepath, obj)
fileContent, err := os.ReadFile(filepath)
if err != nil {
return fmt.Errorf("error reading %q file contents: %v", filepath, err)
}
err = yaml.Unmarshal(fileContent, obj)
if err != nil {
return fmt.Errorf("error unmarshalling %q YAML file: %v", filepath, err)
}
Logf("Successfully loaded YAML file %q into %+v", filepath, obj)
return nil
}