Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

coordinator: ensure hijickRoute's gw is from hostIPRouteForPod #3364

Merged
merged 1 commit into from
Apr 16, 2024
Merged

coordinator: ensure hijickRoute's gw is from hostIPRouteForPod #3364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 40 additions & 26 deletions cmd/coordinator/cmd/command_add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,30 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
return err
}

// get all ip of pod
var allPodIp []netlink.Addr
err = c.netns.Do(func(netNS ns.NetNS) error {
allPodIp, err = networking.GetAllIPAddress(ipFamily, []string{`^lo$`})
if err != nil {
logger.Error("failed to GetAllIPAddress in pod", zap.Error(err))
return fmt.Errorf("failed to GetAllIPAddress in pod: %v", err)
}
return nil
})
if err != nil {
logger.Error("failed to all ip of pod", zap.Error(err))
return err
}
logger.Debug(fmt.Sprintf("all pod ip: %+v", allPodIp))

// get ip addresses of the node
c.hostIPRouteForPod, err = GetAllHostIPRouteForPod(c, ipFamily, allPodIp)
if err != nil {
logger.Error("failed to get IPAddressOnNode", zap.Error(err))
return fmt.Errorf("failed to get IPAddressOnNode: %v", err)
}
logger.Debug(fmt.Sprintf("host IP for route to Pod: %+v", c.hostIPRouteForPod))

// get basic info
switch c.tuneMode {
case ModeUnderlay:
Expand Down Expand Up @@ -238,32 +262,6 @@ func CmdAdd(args *skel.CmdArgs) (err error) {

// =================================

// get all ip of pod
var allPodIp []netlink.Addr
err = c.netns.Do(func(netNS ns.NetNS) error {
allPodIp, err = networking.GetAllIPAddress(ipFamily, []string{`^lo$`})
if err != nil {
logger.Error("failed to GetAllIPAddress in pod", zap.Error(err))
return fmt.Errorf("failed to GetAllIPAddress in pod: %v", err)
}
return nil
})
if err != nil {
logger.Error("failed to all ip of pod", zap.Error(err))
return err
}
logger.Debug(fmt.Sprintf("all pod ip: %+v", allPodIp))

// get ip addresses of the node
c.hostIPRouteForPod, err = GetAllHostIPRouteForPod(c, ipFamily, allPodIp)
if err != nil {
logger.Error("failed to get IPAddressOnNode", zap.Error(err))
return fmt.Errorf("failed to get IPAddressOnNode: %v", err)
}
logger.Debug(fmt.Sprintf("host IP for route to Pod: %+v", c.hostIPRouteForPod))

// =================================

// get ips of this interface(preInterfaceName) from, including ipv4 and ipv6
c.currentAddress, err = networking.IPAddressByName(c.netns, args.IfName, ipFamily)
if err != nil {
Expand Down Expand Up @@ -328,6 +326,22 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
return err
}

// get v4 and v6 gw for hijick route'gw
for _, gw := range c.hostIPRouteForPod {
copy := gw
if copy.To4() != nil {
if c.v4HijackRouteGw == nil && c.ipFamily != netlink.FAMILY_V6 {
c.v4HijackRouteGw = copy
logger.Debug("Get v4HijackRouteGw", zap.String("v4HijackRouteGw", c.v4HijackRouteGw.String()))
}
} else {
if c.v6HijackRouteGw == nil && c.ipFamily != netlink.FAMILY_V4 {
c.v6HijackRouteGw = copy
logger.Debug("Get v6HijackRouteGw", zap.String("v6HijackRouteGw", c.v6HijackRouteGw.String()))
}
}
}

if err = c.setupHijackRoutes(logger, c.currentRuleTable); err != nil {
logger.Error("failed to setupHijackRoutes", zap.Error(err))
return err
Expand Down
34 changes: 11 additions & 23 deletions cmd/coordinator/cmd/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type coordinator struct {
ipFamily, currentRuleTable, hostRuleTable int
tuneMode Mode
hostVethName, podVethName, currentInterface string
v4HijackRouteGw, v6HijackRouteGw net.IP
HijackCIDR, podNics []string
netns, hostNs ns.NetNS
hostVethHwAddress, podVethHwAddress net.HardwareAddr
Expand Down Expand Up @@ -263,15 +264,7 @@ func (c *coordinator) setupNeighborhood(logger *zap.Logger) error {
// setupRoutes setup hijack subnet routes for pod and host
// equivalent to: `ip route add $route table $ruleTable`
func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error {
v4Gw, v6Gw, err := networking.GetGatewayIP(c.currentAddress)
if err != nil {
logger.Error("failed to GetGatewayIP", zap.Error(err))
return err
}

logger.Debug("Debug setupHijackRoutes", zap.String("v4Gw", v4Gw.String()), zap.String("v6Gw", v6Gw.String()))

err = c.netns.Do(func(_ ns.NetNS) error {
err := c.netns.Do(func(_ ns.NetNS) error {
// make sure that veth0/eth0 forwards traffic within the cluster
// eq: ip route add <cluster/service cidr> dev veth0/eth0
for _, hijack := range c.HijackCIDR {
Expand All @@ -283,28 +276,28 @@ func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error

var src *net.IPNet
if nip.To4() != nil {
if v4Gw == nil {
if c.v4HijackRouteGw == nil {
logger.Warn("ignore adding hijack routing table(ipv4), due to ipv4 gateway is nil", zap.String("IPv4 Hijack cidr", hijack))
continue
}
src = c.v4PodOverlayNicAddr
}

if nip.To4() == nil {
if v6Gw == nil {
if c.v6HijackRouteGw == nil {
logger.Warn("ignore adding hijack routing table(ipv6), due to ipv6 gateway is nil", zap.String("IPv6 Hijack cidr", hijack))
continue
}
src = c.v6PodOverlayNicAddr
}

if err := networking.AddRoute(logger, ruleTable, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, ruleTable, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
logger.Error("failed to AddRoute for hijackCIDR", zap.String("Dst", ipNet.String()), zap.Error(err))
return fmt.Errorf("failed to AddRoute for hijackCIDR: %v", err)
}

if c.tuneMode == ModeOverlay && c.firstInvoke {
if err := networking.AddRoute(logger, unix.RT_TABLE_MAIN, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, unix.RT_TABLE_MAIN, c.ipFamily, netlink.SCOPE_UNIVERSE, c.podVethName, src, ipNet, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
logger.Error("failed to AddRoute for hijackCIDR", zap.String("Dst", ipNet.String()), zap.Error(err))
return fmt.Errorf("failed to AddRoute for hijackCIDR: %v", err)
}
Expand Down Expand Up @@ -499,11 +492,6 @@ func (c *coordinator) tunePodRoutes(logger *zap.Logger, configDefaultRouteNIC st
// makeReplyPacketViaVeth make sure that tcp replay packet is forward by veth0
// NOTE: underlay mode only.
func (c *coordinator) makeReplyPacketViaVeth(logger *zap.Logger) error {
v4Gw, v6Gw, err := networking.GetGatewayIP(c.currentAddress)
if err != nil {
return fmt.Errorf("failed to get gateway ips: %v", err)
}

var iptablesInterface []utiliptables.Interface
var ipFamily []int
execer := exec.New()
Expand Down Expand Up @@ -539,7 +527,7 @@ func (c *coordinator) makeReplyPacketViaVeth(logger *zap.Logger) error {
src = c.v6PodOverlayNicAddr
}

if err = networking.AddRoute(logger, c.hostRuleTable, family, netlink.SCOPE_UNIVERSE, c.podVethName, src, nil, v4Gw, v6Gw); err != nil {
if err := networking.AddRoute(logger, c.hostRuleTable, family, netlink.SCOPE_UNIVERSE, c.podVethName, src, nil, c.v4HijackRouteGw, c.v6HijackRouteGw); err != nil {
return err
}
}
Expand Down Expand Up @@ -631,10 +619,10 @@ OUTER1:
}

var DefaultNodeInterfacesToExclude = []string{
"docker.*", "cbr.*", "dummy.*",
"virbr.*", "lxcbr.*", "veth.*", `^lo$`,
`^cali.*`, "flannel.*", "kube-ipvs.*",
"cni.*", "vx-submariner", "cilium*",
"^docker.*", "^cbr.*", "^dummy.*",
"^virbr.*", "^lxcbr.*", "^veth.*", `^lo$`,
`^cali.*`, "^flannel.*", "^kube-ipvs.*",
"^cni.*", "^vx-submariner", "^cilium*",
}

// get additional host ip
Expand Down
Loading