-
Notifications
You must be signed in to change notification settings - Fork 18.6k
/
sandbox_linux.go
372 lines (312 loc) · 9.66 KB
/
sandbox_linux.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
package libnetwork
import (
"context"
"fmt"
"net"
"time"
"github.com/containerd/log"
"github.com/docker/docker/libnetwork/netutils"
"github.com/docker/docker/libnetwork/osl"
"github.com/docker/docker/libnetwork/types"
)
// Linux-specific container configuration flags.
type containerConfigOS struct{} //nolint:nolintlint,unused // only populated on windows
func releaseOSSboxResources(ns *osl.Namespace, ep *Endpoint) {
for _, i := range ns.Interfaces() {
// Only remove the interfaces owned by this endpoint from the sandbox.
if ep.hasInterface(i.SrcName()) {
if err := i.Remove(); err != nil {
log.G(context.TODO()).Debugf("Remove interface %s failed: %v", i.SrcName(), err)
}
}
}
ep.mu.Lock()
joinInfo := ep.joinInfo
vip := ep.virtualIP
lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
ep.mu.Unlock()
if len(vip) > 0 && lbModeIsDSR {
ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}
if err := ns.RemoveAliasIP(ns.GetLoopbackIfaceName(), ipNet); err != nil {
log.G(context.TODO()).WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet)
}
}
if joinInfo == nil {
return
}
// Remove non-interface routes.
for _, r := range joinInfo.StaticRoutes {
if err := ns.RemoveStaticRoute(r); err != nil {
log.G(context.TODO()).Debugf("Remove route failed: %v", err)
}
}
}
// Statistics retrieves the interfaces' statistics for the sandbox.
func (sb *Sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
m := make(map[string]*types.InterfaceStatistics)
sb.mu.Lock()
osb := sb.osSbox
sb.mu.Unlock()
if osb == nil {
return m, nil
}
var err error
for _, i := range osb.Interfaces() {
if m[i.DstName()], err = i.Statistics(); err != nil {
return m, err
}
}
return m, nil
}
func (sb *Sandbox) updateGateway(ep *Endpoint) error {
sb.mu.Lock()
osSbox := sb.osSbox
sb.mu.Unlock()
if osSbox == nil {
return nil
}
osSbox.UnsetGateway() //nolint:errcheck
osSbox.UnsetGatewayIPv6() //nolint:errcheck
if ep == nil {
return nil
}
ep.mu.Lock()
joinInfo := ep.joinInfo
ep.mu.Unlock()
if err := osSbox.SetGateway(joinInfo.gw); err != nil {
return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
}
if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
}
return nil
}
func (sb *Sandbox) ExecFunc(f func()) error {
sb.mu.Lock()
osSbox := sb.osSbox
sb.mu.Unlock()
if osSbox != nil {
return osSbox.InvokeFunc(f)
}
return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID())
}
// SetKey updates the Sandbox Key.
func (sb *Sandbox) SetKey(basePath string) error {
start := time.Now()
defer func() {
log.G(context.TODO()).Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID())
}()
if basePath == "" {
return types.InvalidParameterErrorf("invalid sandbox key")
}
sb.mu.Lock()
if sb.inDelete {
sb.mu.Unlock()
return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id)
}
oldosSbox := sb.osSbox
sb.mu.Unlock()
if oldosSbox != nil {
// If we already have an OS sandbox, release the network resources from that
// and destroy the OS snab. We are moving into a new home further down. Note that none
// of the network resources gets destroyed during the move.
if err := sb.releaseOSSbox(); err != nil {
log.G(context.TODO()).WithError(err).Error("Error destroying os sandbox")
}
}
osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
if err != nil {
return err
}
sb.mu.Lock()
sb.osSbox = osSbox
sb.mu.Unlock()
// If the resolver was setup before stop it and set it up in the
// new osl sandbox.
if oldosSbox != nil && sb.resolver != nil {
sb.resolver.Stop()
if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil {
if err := sb.resolver.Start(); err != nil {
log.G(context.TODO()).Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err)
}
} else {
log.G(context.TODO()).Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err)
}
}
// Set up hosts and resolv.conf files. IPv6 support in the container can't be
// determined yet, as sysctls haven't been applied by the runtime. Calling
// FinishInit after the container task has been created, when sysctls have been
// applied will regenerate these files.
if err := sb.finishInitDNS(); err != nil {
return err
}
for _, ep := range sb.Endpoints() {
if err = sb.populateNetworkResources(ep); err != nil {
return err
}
}
return nil
}
// FinishConfig completes Sandbox configuration. If called after the container task has been
// created, and sysctl settings applied, the configuration will be based on the container's
// IPv6 support.
func (sb *Sandbox) FinishConfig() error {
if sb.config.useDefaultSandBox {
return nil
}
sb.mu.Lock()
osSbox := sb.osSbox
sb.mu.Unlock()
if osSbox == nil {
return nil
}
// If sysctl changes have been made, IPv6 may have been enabled/disabled since last checked.
osSbox.RefreshIPv6LoEnabled()
return sb.finishInitDNS()
}
// IPv6 support can always be determined for host networking. For other network
// types it can only be determined once there's a container namespace to probe,
// return ok=false in that case.
func (sb *Sandbox) ipv6Enabled() (enabled, ok bool) {
// For host networking, IPv6 support depends on the host.
if sb.config.useDefaultSandBox {
return netutils.IsV6Listenable(), true
}
// For other network types, look at whether the container's loopback interface has an IPv6 address.
sb.mu.Lock()
osSbox := sb.osSbox
sb.mu.Unlock()
if osSbox == nil {
return false, false
}
return osSbox.IPv6LoEnabled(), true
}
func (sb *Sandbox) releaseOSSbox() error {
sb.mu.Lock()
osSbox := sb.osSbox
sb.osSbox = nil
sb.mu.Unlock()
if osSbox == nil {
return nil
}
for _, ep := range sb.Endpoints() {
releaseOSSboxResources(osSbox, ep)
}
return osSbox.Destroy()
}
func (sb *Sandbox) restoreOslSandbox() error {
var routes []*types.StaticRoute
// restore osl sandbox
interfaces := make(map[osl.Iface][]osl.IfaceOption)
for _, ep := range sb.endpoints {
ep.mu.Lock()
joinInfo := ep.joinInfo
i := ep.iface
ep.mu.Unlock()
if i == nil {
log.G(context.TODO()).Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID())
continue
}
ifaceOptions := []osl.IfaceOption{
osl.WithIPv4Address(i.addr),
osl.WithRoutes(i.routes),
}
if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
ifaceOptions = append(ifaceOptions, osl.WithIPv6Address(i.addrv6))
}
if i.mac != nil {
ifaceOptions = append(ifaceOptions, osl.WithMACAddress(i.mac))
}
if len(i.llAddrs) != 0 {
ifaceOptions = append(ifaceOptions, osl.WithLinkLocalAddresses(i.llAddrs))
}
interfaces[osl.Iface{SrcName: i.srcName, DstPrefix: i.dstPrefix}] = ifaceOptions
if joinInfo != nil {
routes = append(routes, joinInfo.StaticRoutes...)
}
if ep.needResolver() {
sb.startResolver(true)
}
}
gwep := sb.getGatewayEndpoint()
if gwep == nil {
return nil
}
// restore osl sandbox
return sb.osSbox.Restore(interfaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
}
func (sb *Sandbox) populateNetworkResources(ep *Endpoint) error {
sb.mu.Lock()
if sb.osSbox == nil {
sb.mu.Unlock()
return nil
}
inDelete := sb.inDelete
sb.mu.Unlock()
ep.mu.Lock()
joinInfo := ep.joinInfo
i := ep.iface
lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
ep.mu.Unlock()
if ep.needResolver() {
sb.startResolver(false)
}
if i != nil && i.srcName != "" {
var ifaceOptions []osl.IfaceOption
ifaceOptions = append(ifaceOptions, osl.WithIPv4Address(i.addr), osl.WithRoutes(i.routes))
if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
ifaceOptions = append(ifaceOptions, osl.WithIPv6Address(i.addrv6))
}
if len(i.llAddrs) != 0 {
ifaceOptions = append(ifaceOptions, osl.WithLinkLocalAddresses(i.llAddrs))
}
if i.mac != nil {
ifaceOptions = append(ifaceOptions, osl.WithMACAddress(i.mac))
}
if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
}
if len(ep.virtualIP) > 0 && lbModeIsDSR {
if sb.loadBalancerNID == "" {
if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {
return fmt.Errorf("failed disable ARP for VIP: %v", err)
}
}
ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}
if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)
}
}
}
if joinInfo != nil {
// Set up non-interface routes.
for _, r := range joinInfo.StaticRoutes {
if err := sb.osSbox.AddStaticRoute(r); err != nil {
return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
}
}
}
if ep == sb.getGatewayEndpoint() {
if err := sb.updateGateway(ep); err != nil {
return err
}
}
// Make sure to add the endpoint to the populated endpoint set
// before populating loadbalancers.
sb.mu.Lock()
sb.populatedEndpoints[ep.ID()] = struct{}{}
sb.mu.Unlock()
// Populate load balancer only after updating all the other
// information including gateway and other routes so that
// loadbalancers are populated all the network state is in
// place in the sandbox.
sb.populateLoadBalancers(ep)
// Only update the store if we did not come here as part of
// sandbox delete. If we came here as part of delete then do
// not bother updating the store. The sandbox object will be
// deleted anyway
if !inDelete {
return sb.storeUpdate()
}
return nil
}