Skip to content

Commit

Permalink
Ignore down interfaces when calculating host MTU. (#8496) (#8499)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fasaxc
fasaxc committed Feb 12, 2024
1 parent 5f3f821 commit ac92fd5
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 62 deletions.
7 changes: 6 additions & 1 deletion felix/dataplane/linux/int_dataplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"sync/atomic"
"time"

apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
Expand All @@ -34,6 +33,8 @@ import (
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"k8s.io/client-go/kubernetes"

apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"

"github.com/projectcalico/calico/felix/bpf/bpfmap"
"github.com/projectcalico/calico/felix/bpf/conntrack"
"github.com/projectcalico/calico/felix/bpf/failsafes"
Expand Down Expand Up @@ -1087,6 +1088,10 @@ func findHostMTU(matchRegex *regexp.Regexp) (int, error) {
log.WithFields(fields).Debug("Skipping interface for MTU detection")
continue
}
if !ifacemonitor.LinkIsOperUp(l) {
log.WithFields(fields).Debug("Skipping down interface for MTU detection")
continue
}
log.WithFields(fields).Debug("Examining link for MTU calculation")
if l.Attrs().MTU < smallest || smallest == 0 {
smallest = l.Attrs().MTU
Expand Down
6 changes: 6 additions & 0 deletions felix/fv/infrastructure/topology.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@ func (c *TopologyContainers) Stop() {
}
}

func (c *TopologyContainers) TriggerDelayedStart() {
for _, f := range c.Felixes {
f.TriggerDelayedStart()
}
}

func DefaultTopologyOptions() TopologyOptions {
felixLogLevel := "Info"
if envLogLevel := os.Getenv("FV_FELIX_LOG_LEVEL"); envLogLevel != "" {
Expand Down
145 changes: 87 additions & 58 deletions felix/fv/mtu_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,86 @@ var _ = infrastructure.DatastoreDescribe("VXLAN topology before adding host IPs
client client.Interface
)

expectMTU := func(mtu int) {
for _, felix := range tc.Felixes {
EventuallyWithOffset(1, func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return strings.TrimSpace(out)
}, "60s", "500ms").Should(Equal(fmt.Sprint(mtu)))
}
}

vxlanOverhead := func(ipv6 bool) int {
if ipv6 {
return 70
}
return 50
}

AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
for _, felix := range tc.Felixes {
felix.Exec("ip", "link")
}
infra.DumpErrorData()
}
tc.Stop()
infra.Stop()
})

for _, ipv6 := range []bool{true, false} {
enableIPv6 := ipv6
Describe(fmt.Sprintf("IPv6 enabled: %v", enableIPv6), func() {
Context("with default MTU interface pattern", func() {
BeforeEach(func() {
infra = getInfra()
topologyOptions := infrastructure.DefaultTopologyOptions()
topologyOptions.DelayFelixStart = true
topologyOptions.VXLANMode = api.VXLANModeAlways
topologyOptions.IPIPEnabled = false
topologyOptions.EnableIPv6 = enableIPv6

// Need n>1 or the infra won't set up IP pools!
tc, client = infrastructure.StartNNodeTopology(2, topologyOptions, infra)
})

It("should get the MTU from the main interface", func() {
// Set a distinctive MTU on the main interface.
for _, f := range tc.Felixes {
f.Exec("ip", "link", "set", "dev", "eth0", "mtu", "1312")
}
tc.TriggerDelayedStart()
expectMTU(1312 - vxlanOverhead(enableIPv6))
})

addDummyNIC := func(f *infrastructure.Felix, up bool) {
f.Exec("ip", "link", "add", "type", "dummy")
f.Exec("ip", "link", "set", "dev", "dummy0", "mtu", "1300")
f.Exec("ip", "link", "set", "name", "eth1", "dummy0")
if up {
f.Exec("ip", "link", "set", "dev", "eth1", "up")
}
}
It("should ignore a secondary interface that is down", func() {
// Set a distinctive MTU on the main interface.
for _, f := range tc.Felixes {
f.Exec("ip", "link", "set", "dev", "eth0", "mtu", "1312")
addDummyNIC(f, false)
}
tc.TriggerDelayedStart()
expectMTU(1312 - vxlanOverhead(enableIPv6))
})
It("should consider a secondary interface that is up", func() {
// Set a distinctive MTU on the main interface.
for _, f := range tc.Felixes {
f.Exec("ip", "link", "set", "dev", "eth0", "mtu", "1312")
addDummyNIC(f, true)
}
tc.TriggerDelayedStart()
expectMTU(1300 - vxlanOverhead(enableIPv6))
})
})

Context("with mismatched MTU interface pattern", func() {
BeforeEach(func() {
infra = getInfra()
Expand Down Expand Up @@ -71,39 +148,13 @@ var _ = infrastructure.DatastoreDescribe("VXLAN topology before adding host IPs
}, "30s", "100ms").ShouldNot(HaveOccurred())
})

AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
for _, felix := range tc.Felixes {
felix.Exec("iptables-save", "-c")
felix.Exec("ipset", "list")
felix.Exec("ip", "r")
felix.Exec("ip", "a")
}
}

tc.Stop()

if CurrentGinkgoTestDescription().Failed {
infra.DumpErrorData()
}
infra.Stop()
})

It("should configure MTU correctly based on FelixConfiguration", func() {
// We should NOT detect the primary interface's MTU of 1500, and instead default
// to 1460 due to the mismatched regex. Since VXLAN is enabled, we expect 1460 minus
// the VXLAN overhead of 50 in case of IPv4 or 70 in case of IPv6 to show up in the MTU file.
vxlanDisabledMtu := "1460"
vxlanEnabledMtu := "1410"
if enableIPv6 {
vxlanEnabledMtu = "1390"
}
for _, felix := range tc.Felixes {
Eventually(func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return out
}, "60s", "500ms").Should(ContainSubstring(vxlanEnabledMtu))
}
vxlanDisabledMtu := 1460
vxlanEnabledMtu := vxlanDisabledMtu - vxlanOverhead(enableIPv6)
expectMTU(vxlanEnabledMtu)

// Disable VXLAN. We should expect the MTU to change, but still based on the default 1460.
// Create a default FelixConfiguration
Expand All @@ -123,29 +174,16 @@ var _ = infrastructure.DatastoreDescribe("VXLAN topology before adding host IPs
}

// It should now have an MTU of 1460 since there is no encap.
for _, felix := range tc.Felixes {
Eventually(func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return out
}, "60s", "500ms").Should(ContainSubstring(vxlanDisabledMtu))
}
expectMTU(vxlanDisabledMtu)
})

It("should configure MTU correctly based on IP pools", func() {
// We should NOT detect the primary interface's MTU of 1500, and instead default
// to 1460 due to the mismatched regex. Since VXLAN is enabled, we expect 1460
// minus the VXLAN overhead of 50 in case of IPv4 or 70 in case of IPv6 to show up in the MTU file.
vxlanDisabledMtu := "1460"
vxlanEnabledMtu := "1410"
if enableIPv6 {
vxlanEnabledMtu = "1390"
}
for _, felix := range tc.Felixes {
Eventually(func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return out
}, "60s", "500ms").Should(ContainSubstring(vxlanEnabledMtu))
}
vxlanDisabledMtu := 1460
vxlanEnabledMtu := vxlanDisabledMtu - vxlanOverhead(enableIPv6)
expectMTU(vxlanEnabledMtu)

// Unset VXLAN on FelixConfiguration. We should expect the MTU not to change, since the VXLAN encap is set in the default IPPool.
fc := api.NewFelixConfiguration()
Expand All @@ -155,12 +193,8 @@ var _ = infrastructure.DatastoreDescribe("VXLAN topology before adding host IPs
Expect(err).NotTo(HaveOccurred())

// It should still have an MTU of 1410 (or 1390 for IPv6) since the VXLAN encap is set in the default IPPool.
for _, felix := range tc.Felixes {
Eventually(func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return out
}, "60s", "500ms").Should(ContainSubstring(vxlanEnabledMtu))
}

expectMTU(vxlanEnabledMtu)

// Set VXLANModeNever on the default IP pool(s)
pool, err := client.IPPools().Get(context.Background(), infrastructure.DefaultIPPoolName, options.GetOptions{})
Expand All @@ -177,12 +211,7 @@ var _ = infrastructure.DatastoreDescribe("VXLAN topology before adding host IPs
}

// It should now have an MTU of 1460 since there is no encap.
for _, felix := range tc.Felixes {
Eventually(func() string {
out, _ := felix.ExecOutput("cat", "/var/lib/calico/mtu")
return out
}, "60s", "500ms").Should(ContainSubstring(vxlanDisabledMtu))
}
expectMTU(vxlanDisabledMtu)
})
})
})
Expand Down
4 changes: 2 additions & 2 deletions felix/ifacemonitor/iface_monitor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ func (m *InterfaceMonitor) storeAndNotifyLink(ifaceExists bool, link netlink.Lin
m.storeAndNotifyLinkInner(ifaceExists, newName, link)
}

func linkIsOperUp(link netlink.Link) bool {
func LinkIsOperUp(link netlink.Link) bool {
// We need the operstate of the interface; this is carried in the IFF_RUNNING flag. The
// IFF_UP flag contains the admin state, which doesn't tell us whether we can program routes
// etc.
Expand Down Expand Up @@ -315,7 +315,7 @@ func (m *InterfaceMonitor) storeAndNotifyLinkInner(ifaceExists bool, ifaceName s
}
newState := StateNotPresent
if ifaceExists {
if linkIsOperUp(link) {
if LinkIsOperUp(link) {
newState = StateUp
} else {
newState = StateDown
Expand Down
2 changes: 1 addition & 1 deletion felix/ifacemonitor/update_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ mainLoop:
return
}
idx := int(linkUpd.Index)
linkIsUp := linkUpd.Header.Type == syscall.RTM_NEWLINK && linkIsOperUp(linkUpd.Link)
linkIsUp := linkUpd.Header.Type == syscall.RTM_NEWLINK && LinkIsOperUp(linkUpd.Link)
var delay time.Duration
if linkIsUp {
if len(updatesByIfaceIdx[idx]) == 0 {
Expand Down

0 comments on commit ac92fd5

Please sign in to comment.