Support dual IP stacks in kubernetes CNI plugin

Signed-off-by: Yuva Shankar <yuva29@users.noreply.github.com>

drivers/endpointstate.go

@@ -31,6 +31,7 @@ type OperEndpointState struct {
 	ServiceName string `json:"serviceName"`
 	ContUUID    string `json:"contUUID"`
 	IPAddress   string `json:"ipAddress"`
+	IPv6Address string `json:"ipv6Address"`
 	MacAddress  string `json:"macAddress"`
 	HomingHost  string `json:"homingHost"`
 	IntfName    string `json:"intfName"`
@@ -43,6 +44,7 @@ func (s *OperEndpointState) Matches(c *mastercfg.CfgEndpointState) bool {
 	return s.NetID == c.NetID &&
 		s.EndpointID == c.EndpointID &&
 		s.IPAddress == c.IPAddress &&
+		s.IPv6Address == c.IPv6Address &&
 		s.MacAddress == c.MacAddress &&
 		s.HomingHost == c.HomingHost &&
 		s.IntfName == c.IntfName &&

drivers/ovsd/ovsdriver.go

@@ -451,6 +451,7 @@ func (d *OvsDriver) CreateEndpoint(id string) error {
 		EndpointID:  cfgEp.EndpointID,
 		ServiceName: cfgEp.ServiceName,
 		IPAddress:   cfgEp.IPAddress,
+		IPv6Address: cfgEp.IPv6Address,
 		MacAddress:  cfgEp.MacAddress,
 		IntfName:    cfgEp.IntfName,
 		PortName:    intfName,

mgmtfn/k8splugin/cniapi/api.go

@@ -40,9 +40,10 @@ type CNIPodAttr struct {
 
 // RspAddPod contains the response to the AddPod
 type RspAddPod struct {
-	Result     uint   `json:"result,omitempty"`
-	EndpointID string `json:"endpointid,omitempty"`
-	IPAddress  string `json:"ipaddress,omitempty"`
-	ErrMsg     string `json:"errmsg,omitempty"`
-	ErrInfo    string `json:"errinfo,omitempty"`
+	Result      uint   `json:"result,omitempty"`
+	EndpointID  string `json:"endpointid,omitempty"`
+	IPAddress   string `json:"ipaddress,omitempty"`
+	IPv6Address string `json:"ipv6address,omitempty"`
+	ErrMsg      string `json:"errmsg,omitempty"`
+	ErrInfo     string `json:"errinfo,omitempty"`
 }

mgmtfn/k8splugin/contivk8s/k8s_cni.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"net"
 	"os"
 	"strings"
 
@@ -27,14 +28,20 @@ import (
 	"github.com/contiv/netplugin/version"
 
 	logger "github.com/Sirupsen/logrus"
+	ip "github.com/appc/cni/pkg/ip"
+	cni "github.com/appc/cni/pkg/plugin"
 )
 
+// CNIResponse response format expected from CNI plugins(version 0.1.0)
+type CNIResponse struct {
+	CNIVersion string `json:"cniVersion"`
+	cni.Result
+}
+
 //CNIError : return format from CNI plugin
 type CNIError struct {
 	CNIVersion string `json:"cniVersion"`
-	Code       uint   `json:"code"`
-	Msg        string `json:"msg"`
-	Details    string `json:"details,omitempty"`
+	cni.Error
 }
 
 var log *logger.Entry
@@ -91,9 +98,42 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
 
 	log.Infof("EP created IP: %s\n", result.IPAddress)
 	// Write the ip address of the created endpoint to stdout
-	fmt.Printf("{\n\"cniVersion\": \"0.1.0\",\n")
-	fmt.Printf("\"ip4\": {\n")
-	fmt.Printf("\"ip\": \"%s\"\n}\n}\n", result.IPAddress)
+
+	// ParseCIDR returns a reference to IPNet
+	ip4Net, err := ip.ParseCIDR(result.IPAddress)
+	if err != nil {
+		log.Errorf("Failed to parse IPv4 CIDR: %v", err)
+		return
+	}
+
+	out := CNIResponse{
+		CNIVersion: "0.1.0",
+	}
+
+	out.IP4 = &cni.IPConfig{
+		IP: net.IPNet{IP: ip4Net.IP, Mask: ip4Net.Mask},
+	}
+
+	if result.IPv6Address != "" {
+		ip6Net, err := ip.ParseCIDR(result.IPv6Address)
+		if err != nil {
+			log.Errorf("Failed to parse IPv6 CIDR: %v", err)
+			return
+		}
+
+		out.IP6 = &cni.IPConfig{
+			IP: net.IPNet{IP: ip6Net.IP, Mask: ip6Net.Mask},
+		}
+	}
+
+	data, err := json.MarshalIndent(out, "", "    ")
+	if err != nil {
+		log.Errorf("Failed to marshal json: %v", err)
+		return
+	}
+
+	log.Infof("Response from CNI executable: \n%s", fmt.Sprintf("%s", data))
+	fmt.Printf(fmt.Sprintf("%s", data))
 }
 
 func deletePodFromContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {

mgmtfn/k8splugin/driver.go

@@ -48,9 +48,11 @@ type epSpec struct {
 
 // epAttr contains the assigned attributes of the created ep
 type epAttr struct {
-	IPAddress string
-	PortName  string
-	Gateway   string
+	IPAddress   string
+	PortName    string
+	Gateway     string
+	IPv6Address string
+	IPv6Gateway string
 }
 
 // netdGetEndpoint is a utility that reads the EP oper state
@@ -152,6 +154,7 @@ func createEP(req *epSpec) (*epAttr, error) {
 		return nil, err
 	}
 
+	// this response should contain IPv6 if the underlying network is configured with IPv6
 	log.Infof("Got endpoint create resp from master: %+v", mresp)
 
 	// Ask netplugin to create the endpoint
@@ -181,6 +184,11 @@ func createEP(req *epSpec) (*epAttr, error) {
 	epResponse.IPAddress = ep.IPAddress + "/" + strconv.Itoa(int(nw.SubnetLen))
 	epResponse.Gateway = nw.Gateway
 
+	if ep.IPv6Address != "" {
+		epResponse.IPv6Address = ep.IPv6Address + "/" + strconv.Itoa(int(nw.IPv6SubnetLen))
+		epResponse.IPv6Gateway = nw.IPv6Gateway
+	}
+
 	return &epResponse, nil
 }
 
@@ -237,8 +245,7 @@ func moveToNS(pid int, ifname string) error {
 }
 
 // setIfAttrs sets the required attributes for the container interface
-func setIfAttrs(pid int, ifname, cidr, newname string) error {
-
+func setIfAttrs(pid int, ifname, cidr, cidr6, newname string) error {
 	nsenterPath, err := osexec.LookPath("nsenter")
 	if err != nil {
 		return err
@@ -285,6 +292,17 @@ func setIfAttrs(pid int, ifname, cidr, newname string) error {
 	}
 	log.Infof("Output from ip assign: %v", assignIP)
 
+	if cidr6 != "" {
+		out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
+			"-6", "address", "add", cidr6, "dev", newname).CombinedOutput()
+		if err != nil {
+			log.Errorf("unable to assign IPv6 %s to %s. Error: %s",
+				cidr6, newname, err)
+			return nil
+		}
+		log.Infof("Output of IPv6 assign: %v", out)
+	}
+
 	// Finally, mark the link up
 	bringUp, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
 		"link", "set", "dev", newname, "up").CombinedOutput()
@@ -324,7 +342,7 @@ func addStaticRoute(pid int, subnet, intfName string) error {
 }
 
 // setDefGw sets the default gateway for the container namespace
-func setDefGw(pid int, gw, intfName string) error {
+func setDefGw(pid int, gw, gw6, intfName string) error {
 	nsenterPath, err := osexec.LookPath("nsenter")
 	if err != nil {
 		return err
@@ -338,10 +356,19 @@ func setDefGw(pid int, gw, intfName string) error {
 	out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath, "add",
 		"default", "gw", gw, intfName).CombinedOutput()
 	if err != nil {
-		log.Errorf("unable to set default gw %s. Error: %s - %s",
-			gw, err, out)
+		log.Errorf("unable to set default gw %s. Error: %s - %s", gw, err, out)
 		return nil
 	}
+
+	if gw6 != "" {
+		out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath,
+			"-6", "add", "default", "gw", gw6, intfName).CombinedOutput()
+		if err != nil {
+			log.Errorf("unable to set default IPv6 gateway %s. Error: %s - %s", gw6, err, out)
+			return nil
+		}
+	}
+
 	return nil
 }
 
@@ -430,13 +457,14 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
 	}
 
 	// Set interface attributes for the new port
-	epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, pInfo.IntfName)
+	epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, ep.IPv6Address, pInfo.IntfName)
 	if epErr != nil {
 		log.Errorf("Error setting interface attributes. Err: %v", epErr)
 		setErrorResp(&resp, "Error setting interface attributes", epErr)
 		return resp, epErr
 	}
 
+	//TODO: Host access needs to be enabled for IPv6
 	// if Gateway is not specified on the nw, use the host gateway
 	gwIntf := pInfo.IntfName
 	gw := ep.Gateway
@@ -446,7 +474,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
 		if err != nil {
 			log.Errorf("Error setting host access. Err: %v", err)
 		} else {
-			err = setIfAttrs(pid, hostIf, hostIP, "host1")
+			err = setIfAttrs(pid, hostIf, hostIP, "", "host1")
 			if err != nil {
 				log.Errorf("Move to pid %d failed", pid)
 			} else {
@@ -465,7 +493,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
 	}
 
 	// Set default gateway
-	epErr = setDefGw(pid, gw, gwIntf)
+	epErr = setDefGw(pid, gw, ep.IPv6Gateway, gwIntf)
 	if epErr != nil {
 		log.Errorf("Error setting default gateway. Err: %v", epErr)
 		setErrorResp(&resp, "Error setting default gateway", epErr)
@@ -474,7 +502,13 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
 
 	resp.Result = 0
 	resp.IPAddress = ep.IPAddress
+
+	if ep.IPv6Address != "" {
+		resp.IPv6Address = ep.IPv6Address
+	}
+
 	resp.EndpointID = pInfo.InfraContainerID
+
 	return resp, nil
 }
 

mgmtfn/k8splugin/driver_test.go

@@ -116,12 +116,14 @@ func (s *NetSetup) TestNetSetup(c *C) {
 	address := "192.168.68.68/24"
 	defGW := "192.168.68.1"
 	staticRoute := "192.168.32.0/24"
+	ipv6Address := "2001::100/100"
+	ipv6Gateway := "2001::1/100"
 
-	if err := setIfAttrs(s.pid, s.ifName, address, newName); err != nil {
+	if err := setIfAttrs(s.pid, s.ifName, address, ipv6Address, newName); err != nil {
 		c.Fatalf("setIfAttrs failed: %v", err)
 	}
 
-	if err := setDefGw(s.pid, defGW, newName); err != nil {
+	if err := setDefGw(s.pid, defGW, ipv6Gateway, newName); err != nil {
 		c.Fatalf("setDefGw failed: %v", err)
 	}
 

test/systemtests/k8setup_test.go

@@ -47,7 +47,7 @@ func (k *kubernetes) newContainer(node *node, containerID, name string, spec con
 	}
 	cont.eth0.ip = out
 
-	out, err = cont.node.exec.getIPv6Addr(cont, "eth0")
+	out, err = k8master.exec.getIPv6Addr(cont, "eth0")
 	if err == nil {
 		cont.eth0.ipv6 = out
 	}
@@ -206,8 +206,19 @@ func (k *kubernetes) getIPAddr(c *container, dev string) (string, error) {
 }
 
 func (k *kubernetes) getIPv6Addr(c *container, dev string) (string, error) {
-	/*FIXME: fix for k8 v6 */
-	return "", nil
+	out, err := k8master.tbnode.RunCommandWithOutput(fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1", c.containerID, dev))
+	if err != nil {
+		logrus.Errorf("Failed to get IPv6 for container %q", c.containerID)
+		logrus.Println(out)
+	}
+
+	parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1)
+	if len(parts) < 2 {
+		return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out)
+	}
+
+	parts = strings.Split(parts[1], "/")
+	return strings.TrimSpace(parts[0]), err
 }
 
 func (k *kubernetes) getMACAddr(c *container, dev string) (string, error) {

test/systemtests/kubeadm_test.go

@@ -56,7 +56,7 @@ func (k *kubePod) newContainer(node *node, containerID, name string, spec contai
 	}
 	cont.eth0.ip = out
 
-	out, err = cont.node.exec.getIPv6Addr(cont, "eth0")
+	out, err = k8sMaster.exec.getIPv6Addr(cont, "eth0")
 	if err == nil {
 		cont.eth0.ipv6 = out
 	}
@@ -213,8 +213,21 @@ func (k *kubePod) getIPAddr(c *container, dev string) (string, error) {
 }
 
 func (k *kubePod) getIPv6Addr(c *container, dev string) (string, error) {
-	/*FIXME: fix for k8 v6 */
-	return "", nil
+	out, err := k8sMaster.tbnode.RunCommandWithOutput(
+		fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1",
+			c.containerID, dev))
+	if err != nil {
+		logrus.Errorf("Failed to get IPv6 for container %q", c.containerID)
+		logrus.Println(out)
+	}
+
+	parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1)
+	if len(parts) < 2 {
+		return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out)
+	}
+
+	parts = strings.Split(parts[1], "/")
+	return strings.TrimSpace(parts[0]), err
 }
 
 func (k *kubePod) getMACAddr(c *container, dev string) (string, error) {