-
Notifications
You must be signed in to change notification settings - Fork 1k
/
sandbox.go
452 lines (386 loc) · 12.6 KB
/
sandbox.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
package sandbox
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/cri-o/cri-o/internal/hostport"
"github.com/cri-o/cri-o/internal/oci"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
)
// DevShmPath is the default system wide shared memory path
const DevShmPath = "/dev/shm"
var (
sbStoppedFilename = "stopped"
sbNetworkStoppedFilename = "network-stopped"
)
// Sandbox contains data surrounding kubernetes sandboxes on the server
type Sandbox struct {
portMappings []*hostport.PortMapping
createdAt time.Time
id string
namespace string
// OCI pod name (eg "<namespace>-<name>-<attempt>")
name string
// Kubernetes pod name (eg, "<name>")
kubeName string
logDir string
containers oci.ContainerStorer
processLabel string
mountLabel string
netns NamespaceIface
ipcns NamespaceIface
utsns NamespaceIface
userns NamespaceIface
shmPath string
cgroupParent string
runtimeHandler string
resolvPath string
hostnamePath string
hostname string
// ipv4 or ipv6 cache
ips []string
seccompProfilePath string
labels fields.Set
annotations map[string]string
infraContainer *oci.Container
metadata *Metadata
nsOpts *NamespaceOption
stopMutex sync.RWMutex
created bool
stopped bool
networkStopped bool
privileged bool
hostNetwork bool
usernsMode string
}
type Metadata struct {
// Pod name of the sandbox.
Name string `json:"name,omitempty"`
// Pod UID of the sandbox.
UID string `json:"uid,omitempty"`
// Pod namespace of the sandbox.
Namespace string `json:"namespace,omitempty"`
// Attempt number of creating the sandbox.
Attempt uint32 `json:"attempt,omitempty"`
}
// DefaultShmSize is the default shm size
const DefaultShmSize = 64 * 1024 * 1024
// ErrIDEmpty is the error returned when the id of the sandbox is empty
var ErrIDEmpty = errors.New("PodSandboxId should not be empty")
// New creates and populates a new pod sandbox
// New sandboxes have no containers, no infra container, and no network namespaces associated with them
// An infra container must be attached before the sandbox is added to the state
func New(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *Metadata, shmPath, cgroupParent string, privileged bool, runtimeHandler, resolvPath, hostname string, portMappings []*hostport.PortMapping, hostNetwork bool, createdAt time.Time, usernsMode string) (*Sandbox, error) {
sb := new(Sandbox)
sb.id = id
sb.namespace = namespace
sb.name = name
sb.kubeName = kubeName
sb.logDir = logDir
sb.labels = labels
sb.annotations = annotations
sb.containers = oci.NewMemoryStore()
sb.processLabel = processLabel
sb.mountLabel = mountLabel
sb.metadata = metadata
sb.shmPath = shmPath
sb.cgroupParent = cgroupParent
sb.privileged = privileged
sb.runtimeHandler = runtimeHandler
sb.resolvPath = resolvPath
sb.hostname = hostname
sb.portMappings = portMappings
sb.createdAt = createdAt
sb.hostNetwork = hostNetwork
sb.usernsMode = usernsMode
return sb, nil
}
func (s *Sandbox) CreatedAt() time.Time {
return s.createdAt
}
// SetSeccompProfilePath sets the seccomp profile path
func (s *Sandbox) SetSeccompProfilePath(pp string) {
s.seccompProfilePath = pp
}
// SeccompProfilePath returns the seccomp profile path
func (s *Sandbox) SeccompProfilePath() string {
return s.seccompProfilePath
}
// AddIPs stores the ip in the sandbox
func (s *Sandbox) AddIPs(ips []string) {
s.ips = ips
}
// SetNamespaceOptions sets whether the pod is running using host network
func (s *Sandbox) SetNamespaceOptions(nsOpts *NamespaceOption) {
s.nsOpts = nsOpts
}
// NamespaceOptions returns the namespace options for the sandbox
func (s *Sandbox) NamespaceOptions() *NamespaceOption {
return s.nsOpts
}
// StopMutex returns the mutex to use when stopping the sandbox
func (s *Sandbox) StopMutex() *sync.RWMutex {
return &s.stopMutex
}
// IPs returns the ip of the sandbox
func (s *Sandbox) IPs() []string {
return s.ips
}
// ID returns the id of the sandbox
func (s *Sandbox) ID() string {
return s.id
}
// UsernsMode returns the mode for setting the user namespace, if any.
func (s *Sandbox) UsernsMode() string {
return s.usernsMode
}
// Namespace returns the namespace for the sandbox
func (s *Sandbox) Namespace() string {
return s.namespace
}
// Name returns the name of the sandbox
func (s *Sandbox) Name() string {
return s.name
}
// KubeName returns the kubernetes name for the sandbox
func (s *Sandbox) KubeName() string {
return s.kubeName
}
// LogDir returns the location of the logging directory for the sandbox
func (s *Sandbox) LogDir() string {
return s.logDir
}
// Labels returns the labels associated with the sandbox
func (s *Sandbox) Labels() fields.Set {
return s.labels
}
// Annotations returns a list of annotations for the sandbox
func (s *Sandbox) Annotations() map[string]string {
return s.annotations
}
// Containers returns the ContainerStorer that contains information on all
// of the containers in the sandbox
func (s *Sandbox) Containers() oci.ContainerStorer {
return s.containers
}
// ProcessLabel returns the process label for the sandbox
func (s *Sandbox) ProcessLabel() string {
return s.processLabel
}
// MountLabel returns the mount label for the sandbox
func (s *Sandbox) MountLabel() string {
return s.mountLabel
}
// Metadata returns a set of metadata about the sandbox
func (s *Sandbox) Metadata() *Metadata {
return s.metadata
}
// ShmPath returns the shm path of the sandbox
func (s *Sandbox) ShmPath() string {
return s.shmPath
}
// CgroupParent returns the cgroup parent of the sandbox
func (s *Sandbox) CgroupParent() string {
return s.cgroupParent
}
// Privileged returns whether or not the containers in the sandbox are
// privileged containers
func (s *Sandbox) Privileged() bool {
return s.privileged
}
// RuntimeHandler returns the name of the runtime handler that should be
// picked from the list of runtimes. The name must match the key from the
// map of runtimes.
func (s *Sandbox) RuntimeHandler() string {
return s.runtimeHandler
}
// HostNetwork returns whether the sandbox runs in the host network namespace
func (s *Sandbox) HostNetwork() bool {
return s.hostNetwork
}
// ResolvPath returns the resolv path for the sandbox
func (s *Sandbox) ResolvPath() string {
return s.resolvPath
}
// AddHostnamePath adds the hostname path to the sandbox
func (s *Sandbox) AddHostnamePath(hostname string) {
s.hostnamePath = hostname
}
// HostnamePath retrieves the hostname path from a sandbox
func (s *Sandbox) HostnamePath() string {
return s.hostnamePath
}
// Hostname returns the hostname of the sandbox
func (s *Sandbox) Hostname() string {
return s.hostname
}
// PortMappings returns a list of port mappings between the host and the sandbox
func (s *Sandbox) PortMappings() []*hostport.PortMapping {
return s.portMappings
}
// AddContainer adds a container to the sandbox
func (s *Sandbox) AddContainer(c *oci.Container) {
s.containers.Add(c.Name(), c)
}
// GetContainer retrieves a container from the sandbox
func (s *Sandbox) GetContainer(name string) *oci.Container {
return s.containers.Get(name)
}
// RemoveContainer deletes a container from the sandbox
func (s *Sandbox) RemoveContainer(c *oci.Container) {
s.containers.Delete(c.Name())
}
// SetInfraContainer sets the infrastructure container of a sandbox
// Attempts to set the infrastructure container after one is already present will throw an error
func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error {
if s.infraContainer != nil {
return fmt.Errorf("sandbox already has an infra container")
} else if infraCtr == nil {
return fmt.Errorf("must provide non-nil infra container")
}
s.infraContainer = infraCtr
return nil
}
// InfraContainer returns the infrastructure container for the sandbox
func (s *Sandbox) InfraContainer() *oci.Container {
return s.infraContainer
}
// RemoveInfraContainer removes the infrastructure container of a sandbox
func (s *Sandbox) RemoveInfraContainer() {
s.infraContainer = nil
}
// SetStopped sets the sandbox state to stopped.
// This should be set after a stop operation succeeds
// so that subsequent stops can return fast.
// if createFile is true, it also creates a "stopped" file in the infra container's persistent dir
// this is used to track the sandbox is stopped over reboots
func (s *Sandbox) SetStopped(createFile bool) {
if s.stopped {
return
}
s.stopped = true
if createFile {
if err := s.createFileInInfraDir(sbStoppedFilename); err != nil {
logrus.Errorf("failed to create stopped file in container state. Restore may fail: %v", err)
}
}
}
// Stopped returns whether the sandbox state has been
// set to stopped.
func (s *Sandbox) Stopped() bool {
return s.stopped
}
// SetCreated sets the created status of sandbox to true
func (s *Sandbox) SetCreated() {
s.created = true
}
// NetworkStopped returns whether the network has been stopped
func (s *Sandbox) NetworkStopped() bool {
return s.networkStopped
}
// SetNetworkStopped sets the sandbox network state as stopped
// This should be set after a network stop operation succeeds,
// so we don't double stop the network
// if createFile is true, it creates a "network-stopped" file
// in the infra container's persistent dir
// this is used to track the network is stopped over reboots
// returns an error if an error occurred when creating the network-stopped file
func (s *Sandbox) SetNetworkStopped(createFile bool) error {
if s.networkStopped {
return nil
}
s.networkStopped = true
if createFile {
if err := s.createFileInInfraDir(sbNetworkStoppedFilename); err != nil {
return fmt.Errorf("failed to create state file in container directory. Restores may fail: %v", err)
}
}
return nil
}
func (s *Sandbox) createFileInInfraDir(filename string) error {
// If the sandbox is not yet created,
// this function is being called when
// cleaning up a failed sandbox creation.
// We don't need to create the file, as there will be no
// sandbox to restore
if !s.created {
return nil
}
infra := s.InfraContainer()
f, err := os.Create(filepath.Join(infra.Dir(), filename))
if err == nil {
f.Close()
}
return err
}
func (s *Sandbox) RestoreStopped() {
if s.fileExistsInInfraDir(sbStoppedFilename) {
s.stopped = true
}
if s.fileExistsInInfraDir(sbNetworkStoppedFilename) {
s.networkStopped = true
}
}
func (s *Sandbox) fileExistsInInfraDir(filename string) bool {
infra := s.InfraContainer()
infraFilePath := filepath.Join(infra.Dir(), filename)
if _, err := os.Stat(infraFilePath); err != nil {
if !os.IsNotExist(err) {
logrus.Warnf("error checking if %s exists: %v", infraFilePath, err)
}
return false
}
return true
}
// Created returns the created status of sandbox
func (s *Sandbox) Created() bool {
return s.created
}
// Ready returns whether the sandbox should be marked as ready to the kubelet
// if there is no infra container, it is always considered ready.
// `takeLock` should be set if we need to take the lock to get the infra container's state.
// If there is no infra container, it is never considered ready.
// If the infra container is spoofed, the pod is considered ready when it has been created, but not stopped.
func (s *Sandbox) Ready(takeLock bool) bool {
podInfraContainer := s.InfraContainer()
if podInfraContainer == nil {
return false
}
if podInfraContainer.Spoofed() {
return s.created && !s.stopped
}
// Assume the sandbox is ready, unless it has an infra container that
// isn't running
var cState *oci.ContainerState
if takeLock {
cState = podInfraContainer.State()
} else {
cState = podInfraContainer.StateNoLock()
}
return cState.Status == oci.ContainerStateRunning
}
// UnmountShm removes the shared memory mount for the sandbox and returns an
// error if any failure occurs.
func (s *Sandbox) UnmountShm() error {
fp := s.ShmPath()
if fp == DevShmPath {
return nil
}
// try to unmount, ignoring "not mounted" (EINVAL) error and
// "already unmounted" (ENOENT) error
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil && err != unix.EINVAL && err != unix.ENOENT {
return errors.Wrapf(err, "unable to unmount %s", fp)
}
return nil
}
// NeedsInfra is a function that returns whether the sandbox will need an infra container.
// If the server manages the namespace lifecycles, and the Pid option on the sandbox
// is node or container level, the infra container is not needed
func (s *Sandbox) NeedsInfra(serverDropsInfra bool) bool {
return !serverDropsInfra || s.nsOpts.Pid == NamespaceModePod
}