Modify Pod netrowk setup

Signed-off-by: Yusuke Suzuki <yusuke-suzuki@cybozu.co.jp>
cybozu-go · Sep 14, 2021 · ff17164 · ff17164
1 parent 8c33797
commit ff17164
Showing 1 changed file with 50 additions and 27 deletions.
diff --git a/v2/pkg/nodenet/pod.go b/v2/pkg/nodenet/pod.go
@@ -21,9 +21,10 @@ import (
 var (
 	errNotFound = errors.New("not found")
 
-	hostIPv4    = net.ParseIP("169.254.1.1") // link-local address
-	defaultGWv4 = &net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: net.CIDRMask(0, 32)}
-	defaultGWv6 = &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}
+	hostIPv4      = net.ParseIP("169.254.1.1") // link-local address
+	containerIPv4 = net.ParseIP("169.254.1.2") // link-local address
+	defaultGWv4   = &net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: net.CIDRMask(0, 32)}
+	defaultGWv6   = &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}
 )
 
 // SetupHook is a signature of hook function for PodNetwork.Setup
@@ -197,6 +198,7 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 	defer containerNS.Close()
 
 	// setup veth and configure IP addresses
+	var containerIPv6 net.IP
 	result := &current.Result{}
 	err = containerNS.Do(func(hostNS ns.NetNS) error {
 		vethName := ""
@@ -213,17 +215,30 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 			return fmt.Errorf("netlink: failed to get veth link for container: %w", err)
 		}
 
+		lo, err := netlink.LinkByName("lo")
+		if err != nil {
+			return fmt.Errorf("netlink: failed to get lo: %w", err)
+		}
+
 		idx := 0
 		if conf.IPv4 != nil {
 			ipnet := netlink.NewIPNet(conf.IPv4)
-			err := netlink.AddrAdd(cLink, &netlink.Addr{
+			err := netlink.AddrAdd(lo, &netlink.Addr{
 				IPNet: ipnet,
 				Scope: unix.RT_SCOPE_UNIVERSE,
 			})
 			if err != nil {
 				netlink.LinkDel(cLink)
 				return fmt.Errorf("netlink: failed to add an address: %w", err)
 			}
+			err = netlink.AddrAdd(cLink, &netlink.Addr{
+				IPNet: &net.IPNet{IP: containerIPv4, Mask: net.CIDRMask(30, 32)},
+				Scope: unix.RT_SCOPE_LINK,
+			})
+			if err != nil {
+				netlink.LinkDel(cLink)
+				return fmt.Errorf("netlink: failed to add a link local address: %w", err)
+			}
 			result.IPs = append(result.IPs, &current.IPConfig{
 				Version:   "4",
 				Address:   *ipnet,
@@ -233,7 +248,7 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 
 		if conf.IPv6 != nil {
 			ipnet := netlink.NewIPNet(conf.IPv6)
-			err := netlink.AddrAdd(cLink, &netlink.Addr{
+			err := netlink.AddrAdd(lo, &netlink.Addr{
 				IPNet: ipnet,
 				Scope: unix.RT_SCOPE_UNIVERSE,
 			})
@@ -247,6 +262,11 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 				Address:   *ipnet,
 				Interface: &idx,
 			})
+
+			containerIPv6, err = findLinkLocalIPv6(cLink)
+			if err != nil {
+				return err
+			}
 		}
 
 		result.Interfaces = []*current.Interface{
@@ -289,24 +309,15 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 	var hostIPv6 net.IP
 	if conf.IPv6 != nil {
 		ip.SettleAddresses(hName, 10)
-		v6Addrs, err := netlink.AddrList(hLink, netlink.FAMILY_V6)
+		hostIPv6, err = findLinkLocalIPv6(hLink)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get v6 addresses: %w", err)
-		}
-		for _, a := range v6Addrs {
-			if a.Scope == unix.RT_SCOPE_LINK {
-				hostIPv6 = a.IP
-				break
-			}
-		}
-		if hostIPv6 == nil {
-			return nil, fmt.Errorf("failed to find link-local address of %s", hLink.Attrs().Name)
+			return nil, err
 		}
 
 		err = netlink.RouteAdd(&netlink.Route{
 			Dst:       netlink.NewIPNet(conf.IPv6),
+			Gw:        containerIPv6,
 			LinkIndex: hLink.Attrs().Index,
-			Scope:     netlink.SCOPE_LINK,
 			Protocol:  pn.protocolId,
 			Table:     pn.podTableId,
 		})
@@ -316,7 +327,7 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 	}
 	if conf.IPv4 != nil {
 		err = netlink.AddrAdd(hLink, &netlink.Addr{
-			IPNet: netlink.NewIPNet(hostIPv4),
+			IPNet: &net.IPNet{IP: hostIPv4, Mask: net.CIDRMask(30, 32)},
 			Scope: unix.RT_SCOPE_LINK,
 		})
 		if err != nil {
@@ -325,8 +336,8 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 
 		err = netlink.RouteAdd(&netlink.Route{
 			Dst:       netlink.NewIPNet(conf.IPv4),
+			Gw:        containerIPv4,
 			LinkIndex: hLink.Attrs().Index,
-			Scope:     netlink.SCOPE_LINK,
 			Protocol:  pn.protocolId,
 			Table:     pn.podTableId,
 		})
@@ -342,14 +353,6 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 			return fmt.Errorf("netlink: failed to find link: %w", err)
 		}
 		if conf.IPv4 != nil {
-			err := netlink.RouteAdd(&netlink.Route{
-				Dst:       netlink.NewIPNet(hostIPv4),
-				LinkIndex: l.Attrs().Index,
-				Scope:     netlink.SCOPE_LINK,
-			})
-			if err != nil {
-				return fmt.Errorf("netlink: failed to add route to %s: %w", hostIPv4.String(), err)
-			}
 			err = netlink.RouteAdd(&netlink.Route{
 				Dst:   defaultGWv4,
 				Gw:    hostIPv4,
@@ -384,6 +387,26 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo
 	return result, nil
 }
 
+func findLinkLocalIPv6(link netlink.Link) (net.IP, error) {
+	v6Addrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get v6 addresses: %w", err)
+	}
+
+	var ipv6 net.IP
+	for _, a := range v6Addrs {
+		if a.Scope == unix.RT_SCOPE_LINK {
+			ipv6 = a.IP
+			break
+		}
+	}
+	if ipv6 == nil {
+		return nil, fmt.Errorf("failed to find link-local address of %s", link.Attrs().Name)
+	}
+
+	return ipv6, nil
+}
+
 func (pn *podNetwork) Check(containerId, iface string) error {
 	pn.mu.Lock()
 	defer pn.mu.Unlock()