fix: set vf mac

go.mod

@@ -3,7 +3,7 @@ module github.com/kubeovn/kube-ovn
 go 1.12
 
 require (
-	github.com/Mellanox/sriovnet v1.0.1
+	github.com/Mellanox/sriovnet v1.0.2
 	github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
 	github.com/containernetworking/cni v0.8.0

go.sum

@@ -35,8 +35,8 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Mellanox/sriovnet v1.0.1 h1:g9FqYgcybIuDHKDy8ZEaILeTSXo0r0jzqTFK8rpUSas=
-github.com/Mellanox/sriovnet v1.0.1/go.mod h1:zikbXOU755fnTeF858ym1z4BkQsWYOgW4RpoYCXre/g=
+github.com/Mellanox/sriovnet v1.0.2 h1:VTQHD7OHU6QejTtclt5a2obDfsW1ATRxTCgZmsiKmXI=
+github.com/Mellanox/sriovnet v1.0.2/go.mod h1:pXdSZwahlvP0Xn8nuXcVthBE38Nqf2czo449p5ALLXY=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
@@ -350,6 +350,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.0.0-20160823170715-cfb55aafdaf3/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -432,6 +433,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -490,8 +492,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v0.0.0-20170217164146-9be650865eab/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
+github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -577,6 +580,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

pkg/daemon/ovs.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/Mellanox/sriovnet"
+	sriovutilfs "github.com/Mellanox/sriovnet/pkg/utils/filesystem"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 	goping "github.com/oilbeater/go-ping"
@@ -111,6 +112,16 @@ func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, device
 		if err = netlink.LinkDel(hostLink); err != nil {
 			return fmt.Errorf("delete host link %s failed %v", hostLink, err)
 		}
+	} else {
+		// Ret VF index from PCI
+		vfIndex, err := sriovnet.GetVfIndexByPciAddress(deviceID)
+		if err != nil {
+			klog.Errorf("failed to get vf %s index, %v", deviceID, err)
+			return err
+		}
+		if err = setVfMac(deviceID, vfIndex, "00:00:00:00:00:00"); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -708,17 +719,11 @@ func setupSriovInterface(containerID, deviceID, vfDriver, ifName string, mtu int
 		return "", "", fmt.Errorf("failed to set MTU on %s: %v", hostNicName, err)
 	}
 
-	// 7. set MAC address to VF
-	macAddr, err := net.ParseMAC(mac)
-	if err != nil {
-		return "", "", fmt.Errorf("failed to parse mac %s %v", macAddr, err)
-	}
-	nicLink, err := netlink.LinkByName(uplink)
-	if err != nil {
-		return "", "", fmt.Errorf("can not find nic %s %v", uplink, err)
-	}
-	if err := netlink.LinkSetVfHardwareAddr(nicLink, vfIndex, macAddr); err != nil {
-		return "", "", fmt.Errorf("can not set mac address to vf nic:%s vf:%d %v", uplink, vfIndex, err)
+	if isVfioPciDriver {
+		// 7. set MAC address to VF
+		if err := setVfMac(deviceID, vfIndex, mac); err != nil {
+			return "", "", err
+		}
 	}
 	return hostNicName, vfNetdevice, nil
 }
@@ -848,3 +853,47 @@ func addAdditonalNic(ifName string) error {
 	}
 	return nil
 }
+
+func setVfMac(deviceID string, vfIndex int, mac string) error {
+	macAddr, err := net.ParseMAC(mac)
+	if err != nil {
+		return fmt.Errorf("failed to parse mac %s %v", macAddr, err)
+	}
+
+	pfPci, err := sriovnet.GetPfPciFromVfPci(deviceID)
+	if err != nil {
+		return fmt.Errorf("failed to get pf of device %s %v", deviceID, err)
+	}
+
+	netDevs, err := sriovnet.GetNetDevicesFromPci(pfPci)
+	if err != nil {
+		return fmt.Errorf("failed to get pf of device %s %v", deviceID, err)
+	}
+
+	// get real pf
+	var pfName string
+	for _, dev := range netDevs {
+		devicePortNameFile := filepath.Join(util.NetSysDir, dev, "phys_port_name")
+		physPortName, err := sriovutilfs.Fs.ReadFile(devicePortNameFile)
+		if err != nil {
+			return err
+		}
+
+		if !strings.Contains(strings.TrimSpace(string(physPortName)), "vf") {
+			pfName = dev
+			break
+		}
+	}
+	if pfName == "" {
+		return fmt.Errorf("the PF device was not found in the device list, %v", netDevs)
+	}
+
+	pfLink, err := netlink.LinkByName(pfName)
+	if err != nil {
+		return fmt.Errorf("failed to lookup pf %s: %v", pfName, err)
+	}
+	if err := netlink.LinkSetVfHardwareAddr(pfLink, vfIndex, macAddr); err != nil {
+		return fmt.Errorf("can not set mac address to vf nic:%s vf:%d %v", pfName, vfIndex, err)
+	}
+	return nil
+}

pkg/util/const.go

@@ -116,4 +116,5 @@ const (
 	ChasRetryIntev = 1
 
 	VfioSysDir = "/sys/bus/pci/drivers/vfio-pci"
+	NetSysDir  = "/sys/class/net"
 )