diff --git a/Documentation/cmdref/cilium-agent.md b/Documentation/cmdref/cilium-agent.md index 858849ba8ef2..83392b2cb3e8 100644 --- a/Documentation/cmdref/cilium-agent.md +++ b/Documentation/cmdref/cilium-agent.md @@ -35,6 +35,7 @@ cilium-agent [flags] --bpf-fragments-map-max int Maximum number of entries in fragments tracking map (default 8192) --bpf-map-dynamic-size-ratio float Ratio (0.0-1.0) of total system memory to use for dynamic sizing of CT, NAT and policy BPF maps. Set to 0.0 to disable dynamic BPF map sizing (default: 0.0) --bpf-nat-global-max int Maximum number of entries for the global BPF NAT table (default 524288) + --bpf-neigh-global-max int Maximum number of entries for the global BPF neighbor table (default 524288) --bpf-policy-map-max int Maximum number of entries in endpoint policy map (per endpoint) (default 16384) --bpf-root string Path to BPF filesystem --certificates-directory string Root directory to find certificates specified in L7 TLS policy enforcement (default "/var/run/cilium/certs") diff --git a/bpf/lib/nodeport.h b/bpf/lib/nodeport.h index 63055c3c6460..dcd1cc43a2c4 100644 --- a/bpf/lib/nodeport.h +++ b/bpf/lib/nodeport.h @@ -67,7 +67,7 @@ struct bpf_elf_map __section_maps NODEPORT_NEIGH4 = { .size_key = sizeof(__be32), // ipv4 addr .size_value = sizeof(union macaddr), // hw addr .pinning = PIN_GLOBAL_NS, - .max_elem = SNAT_MAPPING_IPV4_SIZE, + .max_elem = NODEPORT_NEIGH4_SIZE, }; #endif /* ENABLE_IPV4 */ @@ -77,7 +77,7 @@ struct bpf_elf_map __section_maps NODEPORT_NEIGH6 = { .size_key = sizeof(union v6addr), // ipv6 addr .size_value = sizeof(union macaddr), // hw addr .pinning = PIN_GLOBAL_NS, - .max_elem = SNAT_MAPPING_IPV6_SIZE, + .max_elem = NODEPORT_NEIGH6_SIZE, }; /* The IPv6 extension should be 8-bytes aligned */ diff --git a/bpf/node_config.h b/bpf/node_config.h index 287b349070f6..0b645a3158ab 100644 --- a/bpf/node_config.h +++ b/bpf/node_config.h @@ -53,6 +53,7 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x #ifdef ENABLE_NODEPORT #define SNAT_MAPPING_IPV4 test_cilium_snat_v4_external #define SNAT_MAPPING_IPV4_SIZE 524288 +#define NODEPORT_NEIGH4_SIZE 524288 #endif /* ENABLE_NODEPORT */ #endif /* ENABLE_IPV4 */ @@ -60,6 +61,7 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x #ifdef ENABLE_NODEPORT #define SNAT_MAPPING_IPV6 test_cilium_snat_v6_external #define SNAT_MAPPING_IPV6_SIZE 524288 +#define NODEPORT_NEIGH6_SIZE 524288 #endif /* ENABLE_NODEPORT */ #endif /* ENABLE_IPV6 */ diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go index 4b55710d5860..3b8027fa0362 100644 --- a/daemon/cmd/daemon_main.go +++ b/daemon/cmd/daemon_main.go @@ -693,6 +693,9 @@ func init() { flags.Int(option.NATMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF NAT table") option.BindEnv(option.NATMapEntriesGlobalName) + flags.Int(option.NeighMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF neighbor table") + option.BindEnv(option.NeighMapEntriesGlobalName) + flags.Int(option.PolicyMapEntriesName, defaults.PolicyMapEntries, "Maximum number of entries in endpoint policy map (per endpoint)") option.BindEnv(option.PolicyMapEntriesName) diff --git a/install/kubernetes/cilium/charts/config/templates/configmap.yaml b/install/kubernetes/cilium/charts/config/templates/configmap.yaml index 8fd8c31404e3..87fe0a762271 100644 --- a/install/kubernetes/cilium/charts/config/templates/configmap.yaml +++ b/install/kubernetes/cilium/charts/config/templates/configmap.yaml @@ -165,10 +165,14 @@ data: bpf-ct-global-tcp-max: "{{ .Values.global.bpf.ctTcpMax }}" bpf-ct-global-any-max: "{{ .Values.global.bpf.ctAnyMax }}" - # bpf-nat-global-max specified the maximum number of entries in the BPF NAT - # table. + # bpf-nat-global-max specified the maximum number of entries in the + # BPF NAT table. bpf-nat-global-max: "{{ .Values.global.bpf.natMax }}" + # bpf-neigh-global-max specified the maximum number of entries in the + # BPF neighbor table. + bpf-neigh-global-max: "{{ .Values.global.bpf.neighMax }}" + # bpf-policy-map-max specified the maximum number of entries in endpoint # policy map (per endpoint) bpf-policy-map-max: "{{ .Values.global.bpf.policyMapMax }}" diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml index 2127090e5ad9..00d9af6a71db 100644 --- a/install/kubernetes/cilium/values.yaml +++ b/install/kubernetes/cilium/values.yaml @@ -242,6 +242,9 @@ global: # natMax is the maximum number of entries for the NAT table natMax: 524288 + # neighMax is the maximum number of entries for the neighbor table + neighMax: 524288 + # policyMapMax is the maximum number of entries in endpoint policy map (per endpoint) policyMapMax: 16384 diff --git a/install/kubernetes/quick-install.yaml b/install/kubernetes/quick-install.yaml index 25bc52f4c939..a38082a73f65 100644 --- a/install/kubernetes/quick-install.yaml +++ b/install/kubernetes/quick-install.yaml @@ -87,10 +87,14 @@ data: bpf-ct-global-tcp-max: "524288" bpf-ct-global-any-max: "262144" - # bpf-nat-global-max specified the maximum number of entries in the BPF NAT - # table. + # bpf-nat-global-max specified the maximum number of entries in the + # BPF NAT table. bpf-nat-global-max: "524288" + # bpf-neigh-global-max specified the maximum number of entries in the + # BPF neighbor table. + bpf-neigh-global-max: "524288" + # bpf-policy-map-max specified the maximum number of entries in endpoint # policy map (per endpoint) bpf-policy-map-max: "16384" diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go index d452d4a63ce8..01fb75dd5f8f 100644 --- a/pkg/datapath/linux/config/config.go +++ b/pkg/datapath/linux/config/config.go @@ -239,9 +239,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC if option.Config.EnableIPv4 { cDefinesMap["NODEPORT_NEIGH4"] = neighborsmap.Map4Name + cDefinesMap["NODEPORT_NEIGH4_SIZE"] = fmt.Sprintf("%d", option.Config.NeighMapEntriesGlobal) } if option.Config.EnableIPv6 { cDefinesMap["NODEPORT_NEIGH6"] = neighborsmap.Map6Name + cDefinesMap["NODEPORT_NEIGH6_SIZE"] = fmt.Sprintf("%d", option.Config.NeighMapEntriesGlobal) } if option.Config.NodePortMode == option.NodePortModeDSR || option.Config.NodePortMode == option.NodePortModeHybrid { diff --git a/pkg/maps/neighborsmap/neighborsmap.go b/pkg/maps/neighborsmap/neighborsmap.go index eca636ea28da..f0ab35006080 100644 --- a/pkg/maps/neighborsmap/neighborsmap.go +++ b/pkg/maps/neighborsmap/neighborsmap.go @@ -45,7 +45,7 @@ func neighMapsGet() (*bpf.Map, *bpf.Map) { int(unsafe.Sizeof(Key4{})), &Value{}, int(unsafe.Sizeof(Value{})), - option.Config.NATMapEntriesGlobal, + option.Config.NeighMapEntriesGlobal, 0, 0, bpf.ConvertKeyValue, @@ -56,7 +56,7 @@ func neighMapsGet() (*bpf.Map, *bpf.Map) { int(unsafe.Sizeof(Key6{})), &Value{}, int(unsafe.Sizeof(Value{})), - option.Config.NATMapEntriesGlobal, + option.Config.NeighMapEntriesGlobal, 0, 0, bpf.ConvertKeyValue, diff --git a/pkg/option/config.go b/pkg/option/config.go index 9ad84cd4de00..908111d4cbd4 100644 --- a/pkg/option/config.go +++ b/pkg/option/config.go @@ -490,6 +490,9 @@ const ( // NATMapEntriesGlobalName configures max entries for BPF NAT table NATMapEntriesGlobalName = "bpf-nat-global-max" + // NeighMapEntriesGlobalName configures max entries for BPF neighbor table + NeighMapEntriesGlobalName = "bpf-neigh-global-max" + // PolicyMapEntriesName configures max entries for BPF policymap. PolicyMapEntriesName = "bpf-policy-map-max" @@ -789,6 +792,7 @@ var HelpFlagSections = []FlagsSection{ CTMapEntriesTimeoutSVCTCPName, CTMapEntriesTimeoutSVCAnyName, NATMapEntriesGlobalName, + NeighMapEntriesGlobalName, PolicyMapEntriesName, MapEntriesGlobalDynamicSizeRatioName, PreAllocateMapsName, @@ -1303,6 +1307,10 @@ type DaemonConfig struct { // in the BPF NAT table NATMapEntriesGlobal int + // NeighMapEntriesGlobal is the maximum number of neighbor mappings + // allowed in the BPF neigh table + NeighMapEntriesGlobal int + // PolicyMapEntries is the maximum number of peer identities that an // endpoint may allow traffic to exchange traffic with. PolicyMapEntries int @@ -2566,6 +2574,7 @@ func (c *DaemonConfig) calculateBPFMapSizes() error { c.CTMapEntriesGlobalTCP = viper.GetInt(CTMapEntriesGlobalTCPName) c.CTMapEntriesGlobalAny = viper.GetInt(CTMapEntriesGlobalAnyName) c.NATMapEntriesGlobal = viper.GetInt(NATMapEntriesGlobalName) + c.NeighMapEntriesGlobal = viper.GetInt(NeighMapEntriesGlobalName) c.PolicyMapEntries = viper.GetInt(PolicyMapEntriesName) // Don't attempt dynamic sizing if any of the sizeof members was not @@ -2661,6 +2670,15 @@ func (c *DaemonConfig) calculateDynamicBPFMapSizes(totalMemory uint64, dynamicSi } else { log.Debugf("option %s set by user to %v", NATMapEntriesGlobalName, c.NATMapEntriesGlobal) } + if !viper.IsSet(NeighMapEntriesGlobalName) { + // By default we auto-size it to the same value as the NAT map since we + // need to keep at least as many neigh entries. + c.NeighMapEntriesGlobal = c.NATMapEntriesGlobal + log.Debugf("option %s set by dynamic sizing to %v (default %v)", + NeighMapEntriesGlobalName, c.NeighMapEntriesGlobal, NATMapEntriesGlobalDefault) + } else { + log.Debugf("option %s set by user to %v", NeighMapEntriesGlobalName, c.NeighMapEntriesGlobal) + } if !viper.IsSet(PolicyMapEntriesName) { c.PolicyMapEntries = getEntries(defaults.PolicyMapEntries, PolicyMapMin, PolicyMapMax)