Integrate CNI calls into the VMSync flow

Save metadata before we call the CNI plugins, do delete calls based on
the metadata.

Signed-off-by: Roman Mohr <rmohr@redhat.com>

cmd/virt-handler/cni/main/ipamonly/ipamonly.go

@@ -24,6 +24,10 @@ func init() {
 
 type NetConf struct {
 	types.NetConf
+	IPAM struct {
+		Type string `json:"type,omitempty"`
+		Via  string `json:"via,omitempoty"`
+	} `json:"ipam,omitempty"`
 	Master string `json:"master"`
 }
 
@@ -68,6 +72,9 @@ func cmdAdd(args *skel.CmdArgs) error {
 			if err != nil {
 				return fmt.Errorf("error adding route for %v via %v: %v", result, master.Attrs().Name, err)
 			}
+			result.Interfaces = append(result.Interfaces, &current.Interface{Name: n.IPAM.Via})
+		} else if master.Type() == "bridge" {
+			result.Interfaces = append(result.Interfaces, &current.Interface{Name: n.Master})
 		}
 	}
 

cmd/virt-handler/virt-handler.go

@@ -122,14 +122,6 @@ func (app *virtHandlerApp) Run() {
 	// TODO what is scheme used for in Recorder?
 	recorder := broadcaster.NewRecorder(scheme.Scheme, k8sv1.EventSource{Component: "virt-handler", Host: app.HostOverride})
 
-	domainManager, err := virtwrap.NewLibvirtDomainManager(domainConn,
-		recorder,
-		isolation.NewSocketBasedIsolationDetector(app.SocketDir),
-	)
-	if err != nil {
-		panic(err)
-	}
-
 	l, err := labels.Parse(fmt.Sprintf(v1.NodeNameLabel+" in (%s)", app.HostOverride))
 	if err != nil {
 		panic(err)
@@ -148,21 +140,32 @@ func (app *virtHandlerApp) Run() {
 		panic(err)
 	}
 
-	if err := networking.SetNetConfMaster("/etc/cni/net.d", "kubevirt.json", link.Name, link.Name); err != nil {
+	// TODO move the whole  config writes into an extra pod, to make virt-handler node independent
+	if err := networking.SetNetConfMaster("/etc/cni/net.d", "kubevirt.json", link.Name, ""); err != nil {
 		panic(err)
 	}
 	if err := networking.SetNetConfMaster("/etc/cni/net.d", "nodenetwork.json", "kubevirt0", link.Name); err != nil {
 		panic(err)
 	}
 
+	// TODO also do the CNI calls from the extra Pod
 	cnitool := networking.NewCNITool(app.ToolsDir, app.ToolsDir+"/plugins", "/etc/cni/net.d")
-	cnitool.CNIDel("kubevirt", "kubevirt", "kubevirt0", 1)
-	res, err := cnitool.CNIAdd("kubevirt", "kubevirt", "kubevirt0", 1)
+	cnitool.CNIDel("kubevirt", "kubevirt", "kubevirt0", nil, 1)
+	res, err := cnitool.CNIAdd("kubevirt", "kubevirt", "kubevirt0", nil, 1)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(res.String())
 
+	domainManager, err := virtwrap.NewLibvirtDomainManager(domainConn,
+		recorder,
+		isolation.NewSocketBasedIsolationDetector(app.SocketDir),
+		cnitool,
+	)
+	if err != nil {
+		panic(err)
+	}
+
 	configDiskClient := configdisk.NewConfigDiskClient(virtCli)
 
 	// Wire VM controller

pkg/networking/cni.go

@@ -13,8 +13,8 @@ import (
 )
 
 type CNIToolInterface interface {
-	CNIAdd(id string, netConf string, devName string, pid int) (*current.Result, error)
-	CNIDel(id string, netConf string, devName string, pid int) error
+	CNIAdd(id string, netConf string, devName string, mac *string, pid int) (*current.Result, error)
+	CNIDel(id string, netConf string, devName string, mac *string, pid int) error
 }
 
 type cnitool struct {
@@ -27,13 +27,21 @@ func NewCNITool(toolDir string, cniDir string, cniConfDir string) CNIToolInterfa
 	return &cnitool{strings.TrimSuffix(toolDir, "/"), cniDir, cniConfDir}
 }
 
-func (i *cnitool) CNIAdd(id string, netConf string, devName string, pid int) (*current.Result, error) {
-	cmd := exec.Command(i.toolDir+"/cnitool",
+func (i *cnitool) CNIAdd(id string, netConf string, devName string, mac *string, pid int) (*current.Result, error) {
+
+	args := []string{
 		"add", id, netConf, devName,
 		"--from-ns", strconv.Itoa(pid),
 		"--to-ns", strconv.Itoa(pid),
 		"--cni-path", i.cniDir,
-		"--cni-config-path", i.cniConfDir)
+		"--cni-config-path", i.cniConfDir,
+	}
+
+	if mac != nil {
+		args = append(args, "--args", fmt.Sprintf("mac=%s", *mac))
+	}
+
+	cmd := exec.Command(i.toolDir+"/cnitool", args...)
 
 	resp, err := cmd.Output()
 	if err != nil {
@@ -48,13 +56,20 @@ func (i *cnitool) CNIAdd(id string, netConf string, devName string, pid int) (*c
 	return current.NewResultFromResult(res)
 }
 
-func (i *cnitool) CNIDel(id string, netConf string, devName string, pid int) error {
-	cmd := exec.Command(i.toolDir+"/cnitool",
+func (i *cnitool) CNIDel(id string, netConf string, devName string, mac *string, pid int) error {
+
+	args := []string{
 		"del", id, netConf, devName,
 		"--from-ns", strconv.Itoa(pid),
 		"--to-ns", strconv.Itoa(pid),
 		"--cni-path", i.cniDir,
-		"--cni-config-path", i.cniConfDir)
+		"--cni-config-path", i.cniConfDir}
+
+	if mac != nil {
+		args = append(args, "--args", fmt.Sprintf("mac=%s", *mac))
+	}
+
+	cmd := exec.Command(i.toolDir+"/cnitool", args...)
 
 	resp, err := cmd.Output()
 	if err != nil {
@@ -77,7 +92,7 @@ func SetNetConfMaster(cniConfigDir, name, master, via string) error {
 	}
 
 	raw["master"] = master
-	raw["ipam"].(map[string]string)["via"] = via
+	raw["ipam"].(map[string]interface{})["via"] = via
 
 	b, err = json.MarshalIndent(&raw, "", "  ")
 	if err != nil {

pkg/networking/util.go

@@ -10,6 +10,8 @@ import (
 
 	"strconv"
 
+	"crypto/rand"
+
 	"github.com/vishvananda/netlink"
 	"k8s.io/api/core/v1"
 )
@@ -80,3 +82,14 @@ func (i *introspector) GetLinkByIP(ip string, pid int) (*Link, error) {
 func GetNSFromPID(pid uint) string {
 	return fmt.Sprintf("/proc/%d/ns/net", pid)
 }
+
+func RandomMac() (net.HardwareAddr, error) {
+	buf := make([]byte, 6)
+	_, err := rand.Read(buf)
+	if err != nil {
+		return nil, err
+	}
+	// Set the local bit
+	buf[0] |= 2
+	return buf, nil
+}

pkg/virt-handler/virtwrap/api/schema.go

@@ -217,11 +217,12 @@ type DomainSpec struct {
 }
 
 type Metadata struct {
-	Interfaces InterfacesMetadata `xml:"http://kubevirt.io interfaces,omitempty"`
+	Interfaces InterfacesMetadata `xml:"http://kubevirt.io interfaces"`
 }
 
 type InterfaceMetadata struct {
-	Type string `xml:"type"`
+	Type  string `xml:"type"`
+	Index int    `xml:"index"`
 }
 
 type InterfacesMetadata struct {