forked from kata-containers/runtime
-
Notifications
You must be signed in to change notification settings - Fork 0
/
physical_endpoint.go
236 lines (193 loc) · 6.41 KB
/
physical_endpoint.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/hfyeh/runtime/virtcontainers/device/config"
"github.com/hfyeh/runtime/virtcontainers/device/drivers"
persistapi "github.com/hfyeh/runtime/virtcontainers/persist/api"
"github.com/hfyeh/runtime/virtcontainers/pkg/cgroups"
"github.com/safchain/ethtool"
)
// PhysicalEndpoint gathers a physical network interface and its properties
type PhysicalEndpoint struct {
IfaceName string
HardAddr string
EndpointProperties NetworkInfo
EndpointType EndpointType
BDF string
Driver string
VendorDeviceID string
PCIAddr string
}
// Properties returns the properties of the physical interface.
func (endpoint *PhysicalEndpoint) Properties() NetworkInfo {
return endpoint.EndpointProperties
}
// HardwareAddr returns the mac address of the physical network interface.
func (endpoint *PhysicalEndpoint) HardwareAddr() string {
return endpoint.HardAddr
}
// Name returns name of the physical interface.
func (endpoint *PhysicalEndpoint) Name() string {
return endpoint.IfaceName
}
// Type indentifies the endpoint as a physical endpoint.
func (endpoint *PhysicalEndpoint) Type() EndpointType {
return endpoint.EndpointType
}
// PciAddr returns the PCI address of the endpoint.
func (endpoint *PhysicalEndpoint) PciAddr() string {
return endpoint.PCIAddr
}
// SetPciAddr sets the PCI address of the endpoint.
func (endpoint *PhysicalEndpoint) SetPciAddr(pciAddr string) {
endpoint.PCIAddr = pciAddr
}
// SetProperties sets the properties of the physical endpoint.
func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) {
endpoint.EndpointProperties = properties
}
// NetworkPair returns the network pair of the endpoint.
func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair {
return nil
}
// Attach for physical endpoint binds the physical network interface to
// vfio-pci and adds device to the hypervisor with vfio-passthrough.
func (endpoint *PhysicalEndpoint) Attach(s *Sandbox) error {
// Unbind physical interface from host driver and bind to vfio
// so that it can be passed to qemu.
vfioPath, err := bindNICToVFIO(endpoint)
if err != nil {
return err
}
c, err := cgroups.DeviceToCgroupDevice(vfioPath)
if err != nil {
return err
}
d := config.DeviceInfo{
ContainerPath: c.Path,
DevType: string(c.Type),
Major: c.Major,
Minor: c.Minor,
ColdPlug: true,
}
_, err = s.AddDevice(d)
return err
}
// Detach for physical endpoint unbinds the physical network interface from vfio-pci
// and binds it back to the saved host driver.
func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) error {
// Bind back the physical network interface to host.
// We need to do this even if a new network namespace has not
// been created by virtcontainers.
// We do not need to enter the network namespace to bind back the
// physical interface to host driver.
return bindNICToHost(endpoint)
}
// HotAttach for physical endpoint not supported yet
func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error {
return fmt.Errorf("PhysicalEndpoint does not support Hot attach")
}
// HotDetach for physical endpoint not supported yet
func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
return fmt.Errorf("PhysicalEndpoint does not support Hot detach")
}
// isPhysicalIface checks if an interface is a physical device.
// We use ethtool here to not rely on device sysfs inside the network namespace.
func isPhysicalIface(ifaceName string) (bool, error) {
if ifaceName == "lo" {
return false, nil
}
ethHandle, err := ethtool.NewEthtool()
if err != nil {
return false, err
}
defer ethHandle.Close()
bus, err := ethHandle.BusInfo(ifaceName)
if err != nil {
return false, nil
}
// Check for a pci bus format
tokens := strings.Split(bus, ":")
if len(tokens) != 3 {
return false, nil
}
return true, nil
}
var sysPCIDevicesPath = "/sys/bus/pci/devices"
func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) {
// Get ethtool handle to derive driver and bus
ethHandle, err := ethtool.NewEthtool()
if err != nil {
return nil, err
}
defer ethHandle.Close()
// Get BDF
bdf, err := ethHandle.BusInfo(netInfo.Iface.Name)
if err != nil {
return nil, err
}
// Get driver by following symlink /sys/bus/pci/devices/$bdf/driver
driverPath := filepath.Join(sysPCIDevicesPath, bdf, "driver")
link, err := os.Readlink(driverPath)
if err != nil {
return nil, err
}
driver := filepath.Base(link)
// Get vendor and device id from pci space (sys/bus/pci/devices/$bdf)
ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device")
contents, err := ioutil.ReadFile(ifaceDevicePath)
if err != nil {
return nil, err
}
deviceID := strings.TrimSpace(string(contents))
// Vendor id
ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor")
contents, err = ioutil.ReadFile(ifaceVendorPath)
if err != nil {
return nil, err
}
vendorID := strings.TrimSpace(string(contents))
vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID)
vendorDeviceID = strings.TrimSpace(vendorDeviceID)
physicalEndpoint := &PhysicalEndpoint{
IfaceName: netInfo.Iface.Name,
HardAddr: netInfo.Iface.HardwareAddr.String(),
VendorDeviceID: vendorDeviceID,
EndpointType: PhysicalEndpointType,
Driver: driver,
BDF: bdf,
}
return physicalEndpoint, nil
}
func bindNICToVFIO(endpoint *PhysicalEndpoint) (string, error) {
return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
}
func bindNICToHost(endpoint *PhysicalEndpoint) error {
return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
}
func (endpoint *PhysicalEndpoint) save() persistapi.NetworkEndpoint {
return persistapi.NetworkEndpoint{
Type: string(endpoint.Type()),
Physical: &persistapi.PhysicalEndpoint{
BDF: endpoint.BDF,
Driver: endpoint.Driver,
VendorDeviceID: endpoint.VendorDeviceID,
},
}
}
func (endpoint *PhysicalEndpoint) load(s persistapi.NetworkEndpoint) {
endpoint.EndpointType = PhysicalEndpointType
if s.Physical != nil {
endpoint.BDF = s.Physical.BDF
endpoint.Driver = s.Physical.Driver
endpoint.VendorDeviceID = s.Physical.VendorDeviceID
}
}