-
Notifications
You must be signed in to change notification settings - Fork 2.8k
/
xdp.go
278 lines (238 loc) · 8.48 KB
/
xdp.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
package loader
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
"github.com/cilium/ebpf"
"github.com/cilium/ebpf/link"
"github.com/cilium/cilium/pkg/bpf"
"github.com/cilium/cilium/pkg/identity"
"github.com/cilium/cilium/pkg/mac"
"github.com/cilium/cilium/pkg/option"
)
func xdpConfigModeToFlag(xdpMode string) link.XDPAttachFlags {
switch xdpMode {
case option.XDPModeNative, option.XDPModeLinkDriver, option.XDPModeBestEffort:
return link.XDPDriverMode
case option.XDPModeGeneric, option.XDPModeLinkGeneric:
return link.XDPGenericMode
}
return 0
}
// These constant values are returned by the kernel when querying the XDP program attach mode.
// Important: they differ from constants that are used when attaching an XDP program to a netlink device.
const (
XDPAttachedNone uint32 = iota
XDPAttachedDriver
XDPAttachedGeneric
)
// xdpAttachedModeToFlag maps the attach mode that is returned in the metadata when
// querying netlink devices to the attach flags that were used to configure the
// xdp program attachement.
func xdpAttachedModeToFlag(mode uint32) link.XDPAttachFlags {
switch mode {
case XDPAttachedDriver:
return link.XDPDriverMode
case XDPAttachedGeneric:
return link.XDPGenericMode
}
return 0
}
// maybeUnloadObsoleteXDPPrograms removes bpf_xdp.o from previously used
// devices.
//
// bpffsBase is typically set to /sys/fs/bpf/cilium, but can be a temp directory
// during tests.
func maybeUnloadObsoleteXDPPrograms(xdpDevs []string, xdpMode, bpffsBase string) {
links, err := netlink.LinkList()
if err != nil {
log.WithError(err).Warn("Failed to list links for XDP unload")
}
for _, link := range links {
linkxdp := link.Attrs().Xdp
if linkxdp == nil || !linkxdp.Attached {
// No XDP program is attached
continue
}
if strings.Contains(link.Attrs().Name, "cilium") {
// Ignore devices created by cilium-agent
continue
}
used := false
for _, xdpDev := range xdpDevs {
if link.Attrs().Name == xdpDev &&
xdpAttachedModeToFlag(linkxdp.AttachMode) == xdpConfigModeToFlag(xdpMode) {
// XDP mode matches; don't unload, otherwise we might introduce
// intermittent connectivity problems
used = true
break
}
}
if !used {
if err := DetachXDP(link, bpffsBase, symbolFromHostNetdevXDP); err != nil {
log.WithError(err).Warn("Failed to detach obsolete XDP program")
}
}
}
}
// xdpCompileArgs derives compile arguments for bpf_xdp.c.
func xdpCompileArgs(xdpDev string, extraCArgs []string) ([]string, error) {
link, err := netlink.LinkByName(xdpDev)
if err != nil {
return nil, err
}
args := []string{
fmt.Sprintf("-DSECLABEL=%d", identity.ReservedIdentityWorld),
fmt.Sprintf("-DNODE_MAC={.addr=%s}", mac.CArrayString(link.Attrs().HardwareAddr)),
"-DCALLS_MAP=cilium_calls_xdp",
}
args = append(args, extraCArgs...)
if option.Config.EnableNodePort {
args = append(args, []string{
fmt.Sprintf("-DTHIS_MTU=%d", link.Attrs().MTU),
fmt.Sprintf("-DNATIVE_DEV_IFINDEX=%d", link.Attrs().Index),
"-DDISABLE_LOOPBACK_LB",
}...)
}
if option.Config.IsDualStack() {
args = append(args, fmt.Sprintf("-DSECLABEL_IPV4=%d", identity.ReservedIdentityWorldIPv4))
args = append(args, fmt.Sprintf("-DSECLABEL_IPV6=%d", identity.ReservedIdentityWorldIPv6))
} else {
args = append(args, fmt.Sprintf("-DSECLABEL_IPV4=%d", identity.ReservedIdentityWorld))
args = append(args, fmt.Sprintf("-DSECLABEL_IPV6=%d", identity.ReservedIdentityWorld))
}
return args, nil
}
// compileAndLoadXDPProg compiles bpf_xdp.c for the given XDP device and loads it.
func compileAndLoadXDPProg(ctx context.Context, xdpDev, xdpMode string, extraCArgs []string) error {
args, err := xdpCompileArgs(xdpDev, extraCArgs)
if err != nil {
return fmt.Errorf("failed to derive XDP compile extra args: %w", err)
}
dirs := &directoryInfo{
Library: option.Config.BpfDir,
Runtime: option.Config.StateDir,
Output: option.Config.StateDir,
State: option.Config.StateDir,
}
prog := &progInfo{
Source: xdpProg,
Output: xdpObj,
OutputType: outputObject,
Options: args,
}
objPath, err := compile(ctx, prog, dirs)
if err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
progs := []progDefinition{{progName: symbolFromHostNetdevXDP, direction: ""}}
finalize, err := replaceDatapath(ctx, xdpDev, objPath, progs, xdpMode)
if err != nil {
return err
}
finalize()
return err
}
// attachXDPProgram attaches prog with the given progName to link.
//
// bpffsDir should exist and point to the links/ subdirectory in the per-device
// bpffs directory.
func attachXDPProgram(iface netlink.Link, prog *ebpf.Program, progName, bpffsDir string, flags link.XDPAttachFlags) error {
// Attempt to open and update an existing link.
pin := filepath.Join(bpffsDir, progName)
err := bpf.UpdateLink(pin, prog)
switch {
// Update successful, nothing left to do.
case err == nil:
log.Infof("Updated link %s for program %s", pin, progName)
return nil
// Link exists, but is defunct, and needs to be recreated. The program
// no longer gets triggered at this point and the link needs to be removed
// to proceed.
case errors.Is(err, unix.ENOLINK):
if err := os.Remove(pin); err != nil {
return fmt.Errorf("unpinning defunct link %s: %w", pin, err)
}
log.Infof("Unpinned defunct link %s for program %s", pin, progName)
// No existing link found, continue trying to create one.
case errors.Is(err, os.ErrNotExist):
log.Infof("No existing link found at %s for program %s", pin, progName)
default:
return fmt.Errorf("updating link %s for program %s: %w", pin, progName, err)
}
// Create a new link. This will only succeed on nodes that support bpf_link
// and don't have any XDP programs attached through netlink.
l, err := link.AttachXDP(link.XDPOptions{
Program: prog,
Interface: iface.Attrs().Index,
Flags: flags,
})
if err == nil {
defer func() {
// The program was successfully attached using bpf_link. Closing a link
// does not detach the program if the link is pinned.
if err := l.Close(); err != nil {
log.Warnf("Failed to close bpf_link for program %s", progName)
}
}()
if err := l.Pin(pin); err != nil {
return fmt.Errorf("pinning link at %s for program %s : %w", pin, progName, err)
}
// Successfully created and pinned bpf_link.
log.Infof("Program %s attached using bpf_link", progName)
return nil
}
// Kernels before 5.7 don't support bpf_link. In that case link.AttachXDP
// returns ErrNotSupported.
//
// If the kernel supports bpf_link, but an older version of Cilium attached a
// XDP program, link.AttachXDP will return EBUSY.
if !errors.Is(err, unix.EBUSY) && !errors.Is(err, link.ErrNotSupported) {
// Unrecoverable error from AttachRawLink.
return fmt.Errorf("attaching program %s using bpf_link: %w", progName, err)
}
log.Debugf("Performing netlink attach for program %s", progName)
// Omitting XDP_FLAGS_UPDATE_IF_NOEXIST equals running 'ip' with -force,
// and will clobber any existing XDP attachment to the interface, including
// bpf_link attachments created by a different process.
if err := netlink.LinkSetXdpFdWithFlags(iface, prog.FD(), int(flags)); err != nil {
return fmt.Errorf("attaching XDP program %s to interface %s using netlink: %w", progName, iface.Attrs().Name, err)
}
// Nothing left to do, the netlink device now holds a reference to the prog
// the program stays active.
log.Infof("Program %s was attached using netlink", progName)
return nil
}
// DetachXDP removes an XDP program from a network interface.
//
// bpffsBase is typically /sys/fs/bpf/cilium, but can be overridden to a tempdir
// during tests.
func DetachXDP(iface netlink.Link, bpffsBase, progName string) error {
pin := filepath.Join(bpffsDeviceLinksDir(bpffsBase, iface), progName)
err := bpf.UnpinLink(pin)
if err == nil {
return nil
}
if !errors.Is(err, os.ErrNotExist) {
// The pinned link exists, something went wrong unpinning it.
return fmt.Errorf("unpinning XDP program using bpf_link: %w", err)
}
// Pin doesn't exist, fall through to detaching using netlink.
if err := netlink.LinkSetXdpFdWithFlags(iface, -1, int(link.XDPGenericMode)); err != nil {
return fmt.Errorf("detaching generic-mode XDP program using netlink: %w", err)
}
if err := netlink.LinkSetXdpFdWithFlags(iface, -1, int(link.XDPDriverMode)); err != nil {
return fmt.Errorf("detaching driver-mode XDP program using netlink: %w", err)
}
return nil
}