diff --git a/drivers/endpointstate.go b/drivers/endpointstate.go index 3e3895021..0f1c50b6e 100755 --- a/drivers/endpointstate.go +++ b/drivers/endpointstate.go @@ -31,6 +31,7 @@ type OperEndpointState struct { ServiceName string `json:"serviceName"` ContUUID string `json:"contUUID"` IPAddress string `json:"ipAddress"` + IPv6Address string `json:"ipv6Address"` MacAddress string `json:"macAddress"` HomingHost string `json:"homingHost"` IntfName string `json:"intfName"` @@ -43,6 +44,7 @@ func (s *OperEndpointState) Matches(c *mastercfg.CfgEndpointState) bool { return s.NetID == c.NetID && s.EndpointID == c.EndpointID && s.IPAddress == c.IPAddress && + s.IPv6Address == c.IPv6Address && s.MacAddress == c.MacAddress && s.HomingHost == c.HomingHost && s.IntfName == c.IntfName && diff --git a/drivers/ovsd/ovsdriver.go b/drivers/ovsd/ovsdriver.go index 92f236dc4..a4cb56149 100644 --- a/drivers/ovsd/ovsdriver.go +++ b/drivers/ovsd/ovsdriver.go @@ -451,6 +451,7 @@ func (d *OvsDriver) CreateEndpoint(id string) error { EndpointID: cfgEp.EndpointID, ServiceName: cfgEp.ServiceName, IPAddress: cfgEp.IPAddress, + IPv6Address: cfgEp.IPv6Address, MacAddress: cfgEp.MacAddress, IntfName: cfgEp.IntfName, PortName: intfName, diff --git a/mgmtfn/k8splugin/cniapi/api.go b/mgmtfn/k8splugin/cniapi/api.go index 06fce7f4e..0ea533e59 100644 --- a/mgmtfn/k8splugin/cniapi/api.go +++ b/mgmtfn/k8splugin/cniapi/api.go @@ -40,9 +40,10 @@ type CNIPodAttr struct { // RspAddPod contains the response to the AddPod type RspAddPod struct { - Result uint `json:"result,omitempty"` - EndpointID string `json:"endpointid,omitempty"` - IPAddress string `json:"ipaddress,omitempty"` - ErrMsg string `json:"errmsg,omitempty"` - ErrInfo string `json:"errinfo,omitempty"` + Result uint `json:"result,omitempty"` + EndpointID string `json:"endpointid,omitempty"` + IPAddress string `json:"ipaddress,omitempty"` + IPv6Address string `json:"ipv6address,omitempty"` + ErrMsg string `json:"errmsg,omitempty"` + ErrInfo string `json:"errinfo,omitempty"` } diff --git a/mgmtfn/k8splugin/contivk8s/k8s_cni.go b/mgmtfn/k8splugin/contivk8s/k8s_cni.go index c582c5383..9a2998028 100644 --- a/mgmtfn/k8splugin/contivk8s/k8s_cni.go +++ b/mgmtfn/k8splugin/contivk8s/k8s_cni.go @@ -19,6 +19,7 @@ import ( "encoding/json" "flag" "fmt" + "net" "os" "strings" @@ -27,14 +28,20 @@ import ( "github.com/contiv/netplugin/version" logger "github.com/Sirupsen/logrus" + ip "github.com/appc/cni/pkg/ip" + cni "github.com/appc/cni/pkg/plugin" ) +// CNIResponse response format expected from CNI plugins(version 0.1.0) +type CNIResponse struct { + CNIVersion string `json:"cniVersion"` + cni.Result +} + //CNIError : return format from CNI plugin type CNIError struct { CNIVersion string `json:"cniVersion"` - Code uint `json:"code"` - Msg string `json:"msg"` - Details string `json:"details,omitempty"` + cni.Error } var log *logger.Entry @@ -91,9 +98,42 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) { log.Infof("EP created IP: %s\n", result.IPAddress) // Write the ip address of the created endpoint to stdout - fmt.Printf("{\n\"cniVersion\": \"0.1.0\",\n") - fmt.Printf("\"ip4\": {\n") - fmt.Printf("\"ip\": \"%s\"\n}\n}\n", result.IPAddress) + + // ParseCIDR returns a reference to IPNet + ip4Net, err := ip.ParseCIDR(result.IPAddress) + if err != nil { + log.Errorf("Failed to parse IPv4 CIDR: %v", err) + return + } + + out := CNIResponse{ + CNIVersion: "0.1.0", + } + + out.IP4 = &cni.IPConfig{ + IP: net.IPNet{IP: ip4Net.IP, Mask: ip4Net.Mask}, + } + + if result.IPv6Address != "" { + ip6Net, err := ip.ParseCIDR(result.IPv6Address) + if err != nil { + log.Errorf("Failed to parse IPv6 CIDR: %v", err) + return + } + + out.IP6 = &cni.IPConfig{ + IP: net.IPNet{IP: ip6Net.IP, Mask: ip6Net.Mask}, + } + } + + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + log.Errorf("Failed to marshal json: %v", err) + return + } + + log.Infof("Response from CNI executable: \n%s", fmt.Sprintf("%s", data)) + fmt.Printf(fmt.Sprintf("%s", data)) } func deletePodFromContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) { diff --git a/mgmtfn/k8splugin/driver.go b/mgmtfn/k8splugin/driver.go index 3c090b854..dc49c05d0 100644 --- a/mgmtfn/k8splugin/driver.go +++ b/mgmtfn/k8splugin/driver.go @@ -48,9 +48,11 @@ type epSpec struct { // epAttr contains the assigned attributes of the created ep type epAttr struct { - IPAddress string - PortName string - Gateway string + IPAddress string + PortName string + Gateway string + IPv6Address string + IPv6Gateway string } // netdGetEndpoint is a utility that reads the EP oper state @@ -152,6 +154,7 @@ func createEP(req *epSpec) (*epAttr, error) { return nil, err } + // this response should contain IPv6 if the underlying network is configured with IPv6 log.Infof("Got endpoint create resp from master: %+v", mresp) // Ask netplugin to create the endpoint @@ -181,6 +184,11 @@ func createEP(req *epSpec) (*epAttr, error) { epResponse.IPAddress = ep.IPAddress + "/" + strconv.Itoa(int(nw.SubnetLen)) epResponse.Gateway = nw.Gateway + if ep.IPv6Address != "" { + epResponse.IPv6Address = ep.IPv6Address + "/" + strconv.Itoa(int(nw.IPv6SubnetLen)) + epResponse.IPv6Gateway = nw.IPv6Gateway + } + return &epResponse, nil } @@ -237,8 +245,7 @@ func moveToNS(pid int, ifname string) error { } // setIfAttrs sets the required attributes for the container interface -func setIfAttrs(pid int, ifname, cidr, newname string) error { - +func setIfAttrs(pid int, ifname, cidr, cidr6, newname string) error { nsenterPath, err := osexec.LookPath("nsenter") if err != nil { return err @@ -285,6 +292,17 @@ func setIfAttrs(pid int, ifname, cidr, newname string) error { } log.Infof("Output from ip assign: %v", assignIP) + if cidr6 != "" { + out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath, + "-6", "address", "add", cidr6, "dev", newname).CombinedOutput() + if err != nil { + log.Errorf("unable to assign IPv6 %s to %s. Error: %s", + cidr6, newname, err) + return nil + } + log.Infof("Output of IPv6 assign: %v", out) + } + // Finally, mark the link up bringUp, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath, "link", "set", "dev", newname, "up").CombinedOutput() @@ -324,7 +342,7 @@ func addStaticRoute(pid int, subnet, intfName string) error { } // setDefGw sets the default gateway for the container namespace -func setDefGw(pid int, gw, intfName string) error { +func setDefGw(pid int, gw, gw6, intfName string) error { nsenterPath, err := osexec.LookPath("nsenter") if err != nil { return err @@ -338,10 +356,19 @@ func setDefGw(pid int, gw, intfName string) error { out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath, "add", "default", "gw", gw, intfName).CombinedOutput() if err != nil { - log.Errorf("unable to set default gw %s. Error: %s - %s", - gw, err, out) + log.Errorf("unable to set default gw %s. Error: %s - %s", gw, err, out) return nil } + + if gw6 != "" { + out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath, + "-6", "add", "default", "gw", gw6, intfName).CombinedOutput() + if err != nil { + log.Errorf("unable to set default IPv6 gateway %s. Error: %s - %s", gw6, err, out) + return nil + } + } + return nil } @@ -430,13 +457,14 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int } // Set interface attributes for the new port - epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, pInfo.IntfName) + epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, ep.IPv6Address, pInfo.IntfName) if epErr != nil { log.Errorf("Error setting interface attributes. Err: %v", epErr) setErrorResp(&resp, "Error setting interface attributes", epErr) return resp, epErr } + //TODO: Host access needs to be enabled for IPv6 // if Gateway is not specified on the nw, use the host gateway gwIntf := pInfo.IntfName gw := ep.Gateway @@ -446,7 +474,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int if err != nil { log.Errorf("Error setting host access. Err: %v", err) } else { - err = setIfAttrs(pid, hostIf, hostIP, "host1") + err = setIfAttrs(pid, hostIf, hostIP, "", "host1") if err != nil { log.Errorf("Move to pid %d failed", pid) } else { @@ -465,7 +493,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int } // Set default gateway - epErr = setDefGw(pid, gw, gwIntf) + epErr = setDefGw(pid, gw, ep.IPv6Gateway, gwIntf) if epErr != nil { log.Errorf("Error setting default gateway. Err: %v", epErr) setErrorResp(&resp, "Error setting default gateway", epErr) @@ -474,7 +502,13 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int resp.Result = 0 resp.IPAddress = ep.IPAddress + + if ep.IPv6Address != "" { + resp.IPv6Address = ep.IPv6Address + } + resp.EndpointID = pInfo.InfraContainerID + return resp, nil } diff --git a/mgmtfn/k8splugin/driver_test.go b/mgmtfn/k8splugin/driver_test.go index 7568c103d..bd6a2c8d0 100644 --- a/mgmtfn/k8splugin/driver_test.go +++ b/mgmtfn/k8splugin/driver_test.go @@ -116,12 +116,14 @@ func (s *NetSetup) TestNetSetup(c *C) { address := "192.168.68.68/24" defGW := "192.168.68.1" staticRoute := "192.168.32.0/24" + ipv6Address := "2001::100/100" + ipv6Gateway := "2001::1/100" - if err := setIfAttrs(s.pid, s.ifName, address, newName); err != nil { + if err := setIfAttrs(s.pid, s.ifName, address, ipv6Address, newName); err != nil { c.Fatalf("setIfAttrs failed: %v", err) } - if err := setDefGw(s.pid, defGW, newName); err != nil { + if err := setDefGw(s.pid, defGW, ipv6Gateway, newName); err != nil { c.Fatalf("setDefGw failed: %v", err) } diff --git a/test/systemtests/k8setup_test.go b/test/systemtests/k8setup_test.go index 6046340c9..bfbbd6ffb 100755 --- a/test/systemtests/k8setup_test.go +++ b/test/systemtests/k8setup_test.go @@ -47,7 +47,7 @@ func (k *kubernetes) newContainer(node *node, containerID, name string, spec con } cont.eth0.ip = out - out, err = cont.node.exec.getIPv6Addr(cont, "eth0") + out, err = k8master.exec.getIPv6Addr(cont, "eth0") if err == nil { cont.eth0.ipv6 = out } @@ -206,8 +206,19 @@ func (k *kubernetes) getIPAddr(c *container, dev string) (string, error) { } func (k *kubernetes) getIPv6Addr(c *container, dev string) (string, error) { - /*FIXME: fix for k8 v6 */ - return "", nil + out, err := k8master.tbnode.RunCommandWithOutput(fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1", c.containerID, dev)) + if err != nil { + logrus.Errorf("Failed to get IPv6 for container %q", c.containerID) + logrus.Println(out) + } + + parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1) + if len(parts) < 2 { + return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out) + } + + parts = strings.Split(parts[1], "/") + return strings.TrimSpace(parts[0]), err } func (k *kubernetes) getMACAddr(c *container, dev string) (string, error) { diff --git a/test/systemtests/kubeadm_test.go b/test/systemtests/kubeadm_test.go index 09245804f..d0d62c953 100755 --- a/test/systemtests/kubeadm_test.go +++ b/test/systemtests/kubeadm_test.go @@ -56,7 +56,7 @@ func (k *kubePod) newContainer(node *node, containerID, name string, spec contai } cont.eth0.ip = out - out, err = cont.node.exec.getIPv6Addr(cont, "eth0") + out, err = k8sMaster.exec.getIPv6Addr(cont, "eth0") if err == nil { cont.eth0.ipv6 = out } @@ -213,8 +213,21 @@ func (k *kubePod) getIPAddr(c *container, dev string) (string, error) { } func (k *kubePod) getIPv6Addr(c *container, dev string) (string, error) { - /*FIXME: fix for k8 v6 */ - return "", nil + out, err := k8sMaster.tbnode.RunCommandWithOutput( + fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1", + c.containerID, dev)) + if err != nil { + logrus.Errorf("Failed to get IPv6 for container %q", c.containerID) + logrus.Println(out) + } + + parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1) + if len(parts) < 2 { + return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out) + } + + parts = strings.Split(parts[1], "/") + return strings.TrimSpace(parts[0]), err } func (k *kubePod) getMACAddr(c *container, dev string) (string, error) { diff --git a/test/systemtests/network_test.go b/test/systemtests/network_test.go index 9701ab382..5f23f6a11 100755 --- a/test/systemtests/network_test.go +++ b/test/systemtests/network_test.go @@ -111,9 +111,18 @@ func (s *systemtestSuite) testNetworkAddDelete(c *C, encap string) { numContainer = 4 } - for networkNum := 0; networkNum < numContainer/len(s.nodes); networkNum++ { + // this ensures that the IPv6 path is encountered + networkIterations := numContainer / len(s.nodes) + if networkIterations < 2 { + networkIterations = 2 + } + + for networkNum := 0; networkNum < networkIterations; networkNum++ { var v6subnet, v6gateway string - if networkNum%2 == 0 { + + // IPv6 is not supported in `routing` mode. so, + // this `s.fwdMode == "routing"` check is needed here to avoid setting IPv6 + if s.fwdMode == "routing" || networkNum%2 == 0 { v6subnet = "" v6gateway = "" } else { @@ -220,9 +229,18 @@ func (s *systemtestSuite) testNetworkAddDeleteNoGateway(c *C, encap string) { numContainer = 4 } - for networkNum := 0; networkNum < numContainer/len(s.nodes); networkNum++ { + // this ensures that the IPv6 path is encountered + networkIterations := numContainer / len(s.nodes) + if networkIterations < 2 { + networkIterations = 2 + } + + for networkNum := 0; networkNum < networkIterations; networkNum++ { var v6subnet string - if networkNum%2 == 0 { + + // IPv6 is not supported in `routing` mode. so, + // this `s.fwdMode == "routing"` check is needed here to avoid setting IPv6 + if s.fwdMode == "routing" || networkNum%2 == 0 { v6subnet = "" } else { v6subnet = fmt.Sprintf("1001:%d::/120", networkNum) @@ -310,9 +328,18 @@ func (s *systemtestSuite) testNetworkAddDeleteTenant(c *C, encap, fwdmode string c.Assert(s.cli.TenantPost(&client.Tenant{TenantName: tenantName}), IsNil) tenantNames[tenantName] = []string{} - for networkNum := 0; networkNum < numContainer/len(s.nodes); networkNum++ { + // this ensures that the IPv6 path is encountered + networkIterations := numContainer / len(s.nodes) + if networkIterations < 2 { + networkIterations = 2 + } + + for networkNum := 0; networkNum < networkIterations; networkNum++ { var v6subnet, v6gateway string - if networkNum%2 == 0 { + + // IPv6 is not supported in `routing` mode. so, + // this `s.fwdMode == "routing"` check is needed here to avoid setting IPv6 + if fwdmode == "routing" || networkNum%2 == 0 { v6subnet = "" v6gateway = "" } else {