Skip to content

Commit

Permalink
Merge pull request #3358 from cyclinder/coordinator/fix_source_ip
Browse files Browse the repository at this point in the history 
coordinator: ensure hijickRoute's gw is from hostIPRouteForPod
  • Loading branch information
@weizhoublue
weizhoublue committed Apr 16, 2024
2 parents a1a9268 + f7e0192 commit 9125833
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 49 deletions.
66 changes: 40 additions & 26 deletions cmd/coordinator/cmd/command_add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,30 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
return err
}

// get all ip of pod
var allPodIp []netlink.Addr
err = c.netns.Do(func(netNS ns.NetNS) error {
allPodIp, err = networking.GetAllIPAddress(ipFamily, []string{`^lo$`})
if err != nil {
logger.Error("failed to GetAllIPAddress in pod", zap.Error(err))
return fmt.Errorf("failed to GetAllIPAddress in pod: %v", err)
}
return nil
})
if err != nil {
logger.Error("failed to all ip of pod", zap.Error(err))
return err
}
logger.Debug(fmt.Sprintf("all pod ip: %+v", allPodIp))

// get ip addresses of the node
c.hostIPRouteForPod, err = GetAllHostIPRouteForPod(c, ipFamily, allPodIp)
if err != nil {
logger.Error("failed to get IPAddressOnNode", zap.Error(err))
return fmt.Errorf("failed to get IPAddressOnNode: %v", err)
}
logger.Debug(fmt.Sprintf("host IP for route to Pod: %+v", c.hostIPRouteForPod))

// get basic info
switch c.tuneMode {
case ModeUnderlay:
Expand Down Expand Up @@ -238,32 +262,6 @@ func CmdAdd(args *skel.CmdArgs) (err error) {

// =================================

// get all ip of pod
var allPodIp []netlink.Addr
err = c.netns.Do(func(netNS ns.NetNS) error {
allPodIp, err = networking.GetAllIPAddress(ipFamily, []string{`^lo$`})
if err != nil {
logger.Error("failed to GetAllIPAddress in pod", zap.Error(err))
return fmt.Errorf("failed to GetAllIPAddress in pod: %v", err)
}
return nil
})
if err != nil {
logger.Error("failed to all ip of pod", zap.Error(err))
return err
}
logger.Debug(fmt.Sprintf("all pod ip: %+v", allPodIp))

// get ip addresses of the node
c.hostIPRouteForPod, err = GetAllHostIPRouteForPod(c, ipFamily, allPodIp)
if err != nil {
logger.Error("failed to get IPAddressOnNode", zap.Error(err))
return fmt.Errorf("failed to get IPAddressOnNode: %v", err)
}
logger.Debug(fmt.Sprintf("host IP for route to Pod: %+v", c.hostIPRouteForPod))

// =================================

// get ips of this interface(preInterfaceName) from, including ipv4 and ipv6
c.currentAddress, err = networking.IPAddressByName(c.netns, args.IfName, ipFamily)
if err != nil {
Expand Down Expand Up @@ -328,6 +326,22 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
return err
}

// get v4 and v6 gw for hijick route'gw
for _, gw := range c.hostIPRouteForPod {
copy := gw
if copy.To4() != nil {
if c.v4HijackRouteGw == nil && c.ipFamily != netlink.FAMILY_V6 {
c.v4HijackRouteGw = copy
logger.Debug("Get v4HijackRouteGw", zap.String("v4HijackRouteGw", c.v4HijackRouteGw.String()))
}
} else {
if c.v6HijackRouteGw == nil && c.ipFamily != netlink.FAMILY_V4 {
c.v6HijackRouteGw = copy
logger.Debug("Get v6HijackRouteGw", zap.String("v6HijackRouteGw", c.v6HijackRouteGw.String()))
}
}
}

if err = c.setupHijackRoutes(logger, c.currentRuleTable); err != nil {
logger.Error("failed to setupHijackRoutes", zap.Error(err))
return err
Expand Down
34 changes: 11 additions & 23 deletions cmd/coordinator/cmd/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type coordinator struct {
ipFamily, currentRuleTable, hostRuleTable int
tuneMode Mode
hostVethName, podVethName, currentInterface string
v4HijackRouteGw, v6HijackRouteGw net.IP
HijackCIDR, podNics []string
netns, hostNs ns.NetNS
hostVethHwAddress, podVethHwAddress net.HardwareAddr
Expand Down Expand Up @@ -263,15 +264,7 @@ func (c *coordinator) setupNeighborhood(logger *zap.Logger) error {
// setupRoutes setup hijack subnet routes for pod and host
// equivalent to: `ip route add $route table $ruleTable`
func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error {
v4Gw, v6Gw, err := networking.GetGatewayIP(c.currentAddress)
if err != nil {
logger.Error("failed to GetGatewayIP", zap.Error(err))
return err
}

logger.Debug("Debug setupHijackRoutes", zap.String("v4Gw", v4Gw.String()), zap.String("v6Gw", v6Gw.String()))

err = c.netns.Do(func(_ ns.NetNS) error {
err := c.netns.Do(func(_ ns.NetNS) error {
// make sure that veth0/eth0 forwards traffic within the cluster
// eq: ip route add <cluster/service cidr> dev veth0/eth0
for _, hijack := range c.HijackCIDR {
Expand All @@ -283,28 +276,28 @@ func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error

var src *net.IPNet
if nip.To4() != nil {
if v4Gw == nil {
if c.v4HijackRouteGw == nil {
logger.Warn("ignore adding hijack routing table(ipv4), due to ipv4 gateway is nil", zap.String("IPv4 Hijack cidr", hijack))
continue
}
src = c.v4PodOverlayNicAddr
}

if nip.To4() == nil {
if v6Gw == nil {
if c.v6HijackRouteGw == nil {
logger.Warn("ignore adding hijack routing table(ipv6), due to ipv6 gateway is nil", zap.String("IPv6 Hijack cidr", hijack))
continue
}
src = c.v6PodOverlayNicAddr
}

if err := networking.AddRoute(logger, ruleTable, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, ruleTable, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
logger.Error("failed to AddRoute for hijackCIDR", zap.String("Dst", ipNet.String()), zap.Error(err))
return fmt.Errorf("failed to AddRoute for hijackCIDR: %v", err)
}

if c.tuneMode == ModeOverlay && c.firstInvoke {
if err := networking.AddRoute(logger, unix.RT_TABLE_MAIN, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, unix.RT_TABLE_MAIN, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
logger.Error("failed to AddRoute for hijackCIDR", zap.String("Dst", ipNet.String()), zap.Error(err))
return fmt.Errorf("failed to AddRoute for hijackCIDR: %v", err)
}
Expand Down Expand Up @@ -499,11 +492,6 @@ func (c *coordinator) tunePodRoutes(logger *zap.Logger, configDefaultRouteNIC st
// makeReplyPacketViaVeth make sure that tcp replay packet is forward by veth0
// NOTE: underlay mode only.
func (c *coordinator) makeReplyPacketViaVeth(logger *zap.Logger) error {
v4Gw, v6Gw, err := networking.GetGatewayIP(c.currentAddress)
if err != nil {
return fmt.Errorf("failed to get gateway ips: %v", err)
}

var iptablesInterface []utiliptables.Interface
var ipFamily []int
execer := exec.New()
Expand Down Expand Up @@ -539,7 +527,7 @@ func (c *coordinator) makeReplyPacketViaVeth(logger *zap.Logger) error {
src = c.v6PodOverlayNicAddr
}

if err = networking.AddRoute(logger, c.hostRuleTable, family, netlink.SCOPE_UNIVERSE, c.podVethName, src, nil, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, c.hostRuleTable, family, netlink.SCOPE_UNIVERSE, c.podVethName, src, nil, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
return err
}
}
Expand Down Expand Up @@ -631,10 +619,10 @@ OUTER1:
}

var DefaultNodeInterfacesToExclude = []string{
"docker.*", "cbr.*", "dummy.*",
"virbr.*", "lxcbr.*", "veth.*", `^lo$`,
`^cali.*`, "flannel.*", "kube-ipvs.*",
"cni.*", "vx-submariner", "cilium*",
"^docker.*", "^cbr.*", "^dummy.*",
"^virbr.*", "^lxcbr.*", "^veth.*", `^lo$`,
`^cali.*`, "^flannel.*", "^kube-ipvs.*",
"^cni.*", "^vx-submariner", "^cilium*",
}

// get additional host ip
Expand Down

0 comments on commit 9125833

Please sign in to comment.