Skip to content

Commit

Permalink
Reload VF driver only when rdma is enabled
Browse files Browse the repository at this point in the history
  • Loading branch information
@pliurh
pliurh committed Nov 22, 2021
1 parent f4cc0f3 commit 0aee1e5
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 56 deletions.
1 change: 1 addition & 0 deletions api/v1/helper.go
View file
Expand Up @@ -348,6 +348,7 @@ func (p *SriovNetworkNodePolicy) generateVfGroup(iface *InterfaceExt) (*VfGroup,
VfRange: rng,
PolicyName: p.GetName(),
Mtu: p.Spec.Mtu,
IsRdma: p.Spec.IsRdma,
}, nil
}

Expand Down
1 change: 1 addition & 0 deletions api/v1/sriovnetworknodestate_types.go
View file
Expand Up @@ -45,6 +45,7 @@ type VfGroup struct {
VfRange string `json:"vfRange,omitempty"`
PolicyName string `json:"policyName,omitempty"`
Mtu int `json:"mtu,omitempty"`
IsRdma bool `json:"isRdma,omitempty"`
}

type Interfaces []Interface
Expand Down
2 changes: 2 additions & 0 deletions config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml
View file
Expand Up @@ -59,6 +59,8 @@ spec:
properties:
deviceType:
type: string
isRdma:
type: boolean
mtu:
type: integer
policyName:
Expand Down
96 changes: 40 additions & 56 deletions pkg/utils/utils.go
View file
Expand Up @@ -233,13 +233,6 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor
glog.Errorf("configSriovDevice(): fail to set NumVfs for device %s", iface.PciAddress)
return err
}
if strings.EqualFold(iface.LinkType, "IB") {
if err = setVfsGuid(iface); err != nil {
return err
}
} else if err = setVfsAdminMac(ifaceStatus); err != nil {
return err
}
}
// set PF mtu
if iface.Mtu > 0 && iface.Mtu != ifaceStatus.Mtu {
Expand All @@ -255,22 +248,37 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor
if err != nil {
glog.Warningf("configSriovDevice(): unable to parse VFs for device %+v %q", iface.PciAddress, err)
}
pfLink, err := netlink.LinkByName(iface.Name)
if err != nil {
glog.Errorf("setVfsGuid(): unable to get PF link for device %+v %q", iface, err)
return err
}

for _, addr := range vfAddrs {
var group sriovnetworkv1.VfGroup
i := 0
driver := ""
var driver string
var isRdma bool
vfID, err := dputils.GetVFID(addr)
for i, group = range iface.VfGroups {
if err != nil {
glog.Warningf("configSriovDevice(): unable to get VF id %+v %q", iface.PciAddress, err)
}
if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {
isRdma = group.IsRdma
if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) {
driver = group.DeviceType
}
break
}
}
if strings.EqualFold(iface.LinkType, "IB") {
if err = setVfsGuid(addr, pfLink); err != nil {
return err
}
} else if err = setVfsAdminMac(addr, pfLink, isRdma); err != nil {
return err
}
if driver == "" {
if err := BindDefaultDriver(addr); err != nil {
glog.Warningf("configSriovDevice(): fail to bind default driver for device %s", addr)
Expand Down Expand Up @@ -558,35 +566,25 @@ func vfIsReady(pciAddr string) (netlink.Link, error) {
return vfLink, nil
}

func setVfsAdminMac(iface *sriovnetworkv1.InterfaceExt) error {
glog.Infof("setVfsAdminMac(): device %s", iface.PciAddress)
pfLink, err := netlink.LinkByName(iface.Name)
func setVfsAdminMac(vfAddr string, pfLink netlink.Link, isRdma bool) error {
glog.Infof("setVfsAdminMac(): VF %s", vfAddr)

vfID, err := dputils.GetVFID(vfAddr)
if err != nil {
glog.Errorf("setVfsAdminMac(): unable to get PF link for device %+v %q", iface, err)
glog.Errorf("setVfsAdminMac(): unable to get VF id %+v %q", vfAddr, err)
return err
}
vfs, err := dputils.GetVFList(iface.PciAddress)
vfLink, err := vfIsReady(vfAddr)
if err != nil {
glog.Errorf("setVfsAdminMac(): VF link is not ready for device %+v %q", vfAddr, err)
return err
}
for _, addr := range vfs {
vfID, err := dputils.GetVFID(addr)
if err != nil {
glog.Errorf("setVfsAdminMac(): unable to get VF id %+v %q", iface.PciAddress, err)
return err
}
vfLink, err := vfIsReady(addr)
if err != nil {
glog.Errorf("setVfsAdminMac(): VF link is not ready for device %+v %q", addr, err)
return err
}
if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil {
return err
}
if err = Unbind(addr); err != nil {
return err
}
if err = BindDefaultDriver(addr); err != nil {
if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil {
return err
}
if isRdma {
glog.Infof("setVfsAdminMac(): unbind driver for %s", vfAddr)
if err = Unbind(vfAddr); err != nil {
return err
}
}
Expand All @@ -613,36 +611,22 @@ func getLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string {
return ""
}

func setVfsGuid(iface *sriovnetworkv1.Interface) error {
glog.Infof("setVfsGuid(): device %s", iface.PciAddress)
pfLink, err := netlink.LinkByName(iface.Name)
func setVfsGuid(vfAddr string, pfLink netlink.Link) error {
glog.Infof("setVfsGuid(): VF %s", vfAddr)
vfID, err := dputils.GetVFID(vfAddr)
if err != nil {
glog.Errorf("setVfsGuid(): unable to get PF link for device %+v %q", iface, err)
glog.Errorf("setVfsGuid(): unable to get VF id %+v %q", vfAddr, err)
return err
}
vfs, err := dputils.GetVFList(iface.PciAddress)
if err != nil {
guid := generateRandomGuid()
if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
return err
}
for _, addr := range vfs {
vfID, err := dputils.GetVFID(addr)
if err != nil {
glog.Errorf("setVfsGuid(): unable to get VF id %+v %q", iface.PciAddress, err)
return err
}
guid := generateRandomGuid()
if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
return err
}
if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil {
return err
}
if err = Unbind(addr); err != nil {
return err
}
if err = BindDefaultDriver(addr); err != nil {
return err
}
if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil {
return err
}
if err = Unbind(vfAddr); err != nil {
return err
}

return nil
Expand Down

0 comments on commit 0aee1e5

Please sign in to comment.