Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Enable IPv4 GARP and IPv6 NDN by default #179

Closed
wants to merge 1 commit into from
Closed

WIP: Enable IPv4 GARP and IPv6 NDN by default #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions cmd/sriov/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,22 +85,21 @@ func cmdAdd(args *skel.CmdArgs) error {
}}

if !netConf.DPDKMode {
macAddr, err = sm.SetupVF(netConf, args.IfName, args.ContainerID, netns)
err = sm.SetupVF(netConf, args.IfName, netns)
defer func() {
if err != nil {
err := netns.Do(func(_ ns.NetNS) error {
_, err := netlink.LinkByName(args.IfName)
return err
})
if err == nil {
sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns)
sm.ReleaseVF(netConf, args.IfName, netns)
}
}
}()
if err != nil {
return fmt.Errorf("failed to set up pod interface %q from the device %q: %v", args.IfName, netConf.Master, err)
}
result.Interfaces[0].Mac = macAddr
}

// run the IPAM plugin
Expand Down Expand Up @@ -144,6 +143,29 @@ func cmdAdd(args *skel.CmdArgs) error {
result = newResult
}

if !netConf.DPDKMode {
macAddr, err = sm.SetEffectiveMAC(netConf, netns)
if err != nil {
return fmt.Errorf("failed to set effective mac address for interface %q from the device %q: %v", args.IfName, netConf.Master, err)
}
defer func() {
if err != nil {
err := netns.Do(func(_ ns.NetNS) error {
_, err := netlink.LinkByName(args.IfName)
return err
})
if err == nil {
sm.ResetEffectiveMAC(netConf, netns)
sm.ReleaseVF(netConf, args.IfName, netns)
}
}
}()
if err != nil {
return fmt.Errorf("failed to release interface %q from the device %q: %v", args.IfName, netConf.Master, err)
}
result.Interfaces[0].Mac = macAddr
}

// Cache NetConf for CmdDel
if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
return fmt.Errorf("error saving NetConf %q", err)
Expand Down Expand Up @@ -195,7 +217,10 @@ func cmdDel(args *skel.CmdArgs) error {
}
defer netns.Close()

if err = sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns); err != nil {
if err = sm.ResetEffectiveMAC(netConf, netns); err != nil {
return err
}
if err = sm.ReleaseVF(netConf, args.IfName, netns); err != nil {
return err
}
}
Expand Down
123 changes: 82 additions & 41 deletions pkg/sriov/sriov.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ type pciUtils interface {
getSriovNumVfs(ifName string) (int, error)
getVFLinkNamesFromVFID(pfName string, vfID int) ([]string, error)
getPciAddress(ifName string, vf int) (string, error)
enableArpAndNdiscNotify(ifName string) error
}

type pciUtilsImpl struct{}
Expand All @@ -121,10 +122,16 @@ func (p *pciUtilsImpl) getPciAddress(ifName string, vf int) (string, error) {
return utils.GetPciAddress(ifName, vf)
}

func (p *pciUtilsImpl) enableArpAndNdiscNotify(ifName string) error {
return utils.EnableArpAndNdiscNotify(ifName)
}

// Manager provides interface invoke sriov nic related operations
type Manager interface {
SetupVF(conf *sriovtypes.NetConf, podifName string, cid string, netns ns.NetNS) (string, error)
ReleaseVF(conf *sriovtypes.NetConf, podifName string, cid string, netns ns.NetNS) error
SetEffectiveMAC(conf *sriovtypes.NetConf, netns ns.NetNS) (string, error)
ResetEffectiveMAC(conf *sriovtypes.NetConf, netns ns.NetNS) error
SetupVF(conf *sriovtypes.NetConf, podifName string, netns ns.NetNS) error
ReleaseVF(conf *sriovtypes.NetConf, podifName string, netns ns.NetNS) error
ResetVFConfig(conf *sriovtypes.NetConf) error
ApplyVFConfig(conf *sriovtypes.NetConf) error
}
Expand All @@ -142,72 +149,118 @@ func NewSriovManager() Manager {
}
}

// SetEffectiveMac sets VF effective MAC address
func (s *sriovManager) SetEffectiveMAC(conf *sriovtypes.NetConf, netns ns.NetNS) (string, error) {
var macAddress string

if err := netns.Do(func(_ ns.NetNS) error {
linkObj, err := s.nLink.LinkByName(conf.ContIFNames)
if err != nil {
return fmt.Errorf("error getting VF netdevice with name %s", conf.ContIFNames)
}

macAddress = linkObj.Attrs().HardwareAddr.String()
// Set effective MAC address
if conf.MAC != "" {
hwaddr, err := net.ParseMAC(conf.MAC)
if err != nil {
return fmt.Errorf("failed to parse MAC address %s: %v", conf.MAC, err)
}

// Save the original effective MAC address before overriding it
conf.OrigVfState.EffectiveMAC = macAddress

if err = s.nLink.LinkSetHardwareAddr(linkObj, hwaddr); err != nil {
return fmt.Errorf("failed to set netlink MAC address to %s: %v", hwaddr, err)
}
macAddress = conf.MAC
}
return nil
}); err != nil {
return "", fmt.Errorf("error setting up interface in container namespace: %q", err)
}
return macAddress, nil
}

// ResetEffectiveMAC resets VF effective MAC address
func (s *sriovManager) ResetEffectiveMAC(conf *sriovtypes.NetConf, netns ns.NetNS) error {
if err := netns.Do(func(_ ns.NetNS) error {
linkObj, err := s.nLink.LinkByName(conf.ContIFNames)
if err != nil {
return fmt.Errorf("failed to get netlink device with name %s: %q", conf.ContIFNames, err)
}

if conf.MAC != "" && conf.OrigVfState.EffectiveMAC != "" {
hwaddr, err := net.ParseMAC(conf.OrigVfState.EffectiveMAC)
if err != nil {
return fmt.Errorf("failed to parse original effective MAC address %s: %v", conf.OrigVfState.EffectiveMAC, err)
}

if err = s.nLink.LinkSetHardwareAddr(linkObj, hwaddr); err != nil {
return fmt.Errorf("failed to restore original effective netlink MAC address %s: %v", hwaddr, err)
}
}
return nil
}); err != nil {
return fmt.Errorf("error setting up interface in container namespace: %q", err)
}
return nil
}

// SetupVF sets up a VF in Pod netns
func (s *sriovManager) SetupVF(conf *sriovtypes.NetConf, podifName string, cid string, netns ns.NetNS) (string, error) {
func (s *sriovManager) SetupVF(conf *sriovtypes.NetConf, podifName string, netns ns.NetNS) error {
linkName := conf.OrigVfState.HostIFName

linkObj, err := s.nLink.LinkByName(linkName)
if err != nil {
return "", fmt.Errorf("error getting VF netdevice with name %s", linkName)
return fmt.Errorf("error getting VF netdevice with name %s", linkName)
}

// tempName used as intermediary name to avoid name conflicts
tempName := fmt.Sprintf("%s%d", "temp_", linkObj.Attrs().Index)

// 1. Set link down
if err := s.nLink.LinkSetDown(linkObj); err != nil {
return "", fmt.Errorf("failed to down vf device %q: %v", linkName, err)
return fmt.Errorf("failed to down vf device %q: %v", linkName, err)
}

// 2. Set temp name
if err := s.nLink.LinkSetName(linkObj, tempName); err != nil {
return "", fmt.Errorf("error setting temp IF name %s for %s", tempName, linkName)
}

macAddress := linkObj.Attrs().HardwareAddr.String()
// 3. Set MAC address
if conf.MAC != "" {
hwaddr, err := net.ParseMAC(conf.MAC)
if err != nil {
return "", fmt.Errorf("failed to parse MAC address %s: %v", conf.MAC, err)
}

// Save the original effective MAC address before overriding it
conf.OrigVfState.EffectiveMAC = linkObj.Attrs().HardwareAddr.String()

if err = s.nLink.LinkSetHardwareAddr(linkObj, hwaddr); err != nil {
return "", fmt.Errorf("failed to set netlink MAC address to %s: %v", hwaddr, err)
}
macAddress = conf.MAC
return fmt.Errorf("error setting temp IF name %s for %s", tempName, linkName)
}

// 4. Change netns
// 3. Change netns
if err := s.nLink.LinkSetNsFd(linkObj, int(netns.Fd())); err != nil {
return "", fmt.Errorf("failed to move IF %s to netns: %q", tempName, err)
return fmt.Errorf("failed to move IF %s to netns: %q", tempName, err)
}

if err := netns.Do(func(_ ns.NetNS) error {
// 5. Set Pod IF name
// 4. Set Pod IF name
if err := s.nLink.LinkSetName(linkObj, podifName); err != nil {
return fmt.Errorf("error setting container interface name %s for %s", linkName, tempName)
}

// 5. Enable arp notify
if err := s.utils.enableArpAndNdiscNotify(podifName); err != nil {
return fmt.Errorf("failed to enable arp_notify for IF: %s, %q", podifName, err)
}

// 6. Bring IF up in Pod netns
if err := s.nLink.LinkSetUp(linkObj); err != nil {
return fmt.Errorf("error bringing interface up in container ns: %q", err)
}

return nil
}); err != nil {
return "", fmt.Errorf("error setting up interface in container namespace: %q", err)
return fmt.Errorf("error setting up interface in container namespace: %q", err)
}
conf.ContIFNames = podifName

return macAddress, nil
return nil
}

// ReleaseVF reset a VF from Pod netns and return it to init netns
func (s *sriovManager) ReleaseVF(conf *sriovtypes.NetConf, podifName string, cid string, netns ns.NetNS) error {
func (s *sriovManager) ReleaseVF(conf *sriovtypes.NetConf, podifName string, netns ns.NetNS) error {

initns, err := ns.GetCurrentNS()
if err != nil {
Expand Down Expand Up @@ -237,18 +290,6 @@ func (s *sriovManager) ReleaseVF(conf *sriovtypes.NetConf, podifName string, cid
return fmt.Errorf("failed to rename link %s to host name %s: %q", podifName, conf.OrigVfState.HostIFName, err)
}

// reset effective MAC address
if conf.MAC != "" {
hwaddr, err := net.ParseMAC(conf.OrigVfState.EffectiveMAC)
if err != nil {
return fmt.Errorf("failed to parse original effective MAC address %s: %v", conf.OrigVfState.EffectiveMAC, err)
}

if err = s.nLink.LinkSetHardwareAddr(linkObj, hwaddr); err != nil {
return fmt.Errorf("failed to restore original effective netlink MAC address %s: %v", hwaddr, err)
}
}

// move VF device to init netns
if err = s.nLink.LinkSetNsFd(linkObj, int(initns.Fd())); err != nil {
return fmt.Errorf("failed to move interface %s to init netns: %v", conf.OrigVfState.HostIFName, err)
Expand Down
34 changes: 21 additions & 13 deletions pkg/sriov/sriov_test.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading