bpf: make neigh{4,6} maps size configurable from agent and helm

The BPF neigh{4,6} maps are decoupled from SNAT NodePort mode these
days and are used for i) request IP/Mac mappings for remote backends,
ii) for local backends and iii) backend to Mac mappings. Make its size
configurable in order to allow for more flexibility. Default to the
same as NAT map size.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

Documentation/cmdref/cilium-agent.md

@@ -35,6 +35,7 @@ cilium-agent [flags]
       --bpf-fragments-map-max int                     Maximum number of entries in fragments tracking map (default 8192)
       --bpf-map-dynamic-size-ratio float              Ratio (0.0-1.0) of total system memory to use for dynamic sizing of CT, NAT and policy BPF maps. Set to 0.0 to disable dynamic BPF map sizing (default: 0.0)
       --bpf-nat-global-max int                        Maximum number of entries for the global BPF NAT table (default 524288)
+      --bpf-neigh-global-max int                      Maximum number of entries for the global BPF neighbor table (default 524288)
       --bpf-policy-map-max int                        Maximum number of entries in endpoint policy map (per endpoint) (default 16384)
       --bpf-root string                               Path to BPF filesystem
       --certificates-directory string                 Root directory to find certificates specified in L7 TLS policy enforcement (default "/var/run/cilium/certs")

bpf/lib/nodeport.h

@@ -67,7 +67,7 @@ struct bpf_elf_map __section_maps NODEPORT_NEIGH4 = {
 	.size_key	= sizeof(__be32),		// ipv4 addr
 	.size_value	= sizeof(union macaddr),	// hw addr
 	.pinning	= PIN_GLOBAL_NS,
-	.max_elem	= SNAT_MAPPING_IPV4_SIZE,
+	.max_elem	= NODEPORT_NEIGH4_SIZE,
 };
 #endif /* ENABLE_IPV4 */
 
@@ -77,7 +77,7 @@ struct bpf_elf_map __section_maps NODEPORT_NEIGH6 = {
 	.size_key	= sizeof(union v6addr),		// ipv6 addr
 	.size_value	= sizeof(union macaddr),	// hw addr
 	.pinning	= PIN_GLOBAL_NS,
-	.max_elem	= SNAT_MAPPING_IPV6_SIZE,
+	.max_elem	= NODEPORT_NEIGH6_SIZE,
 };
 
 /* The IPv6 extension should be 8-bytes aligned */

bpf/node_config.h

@@ -53,13 +53,15 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x
 #ifdef ENABLE_NODEPORT
 #define SNAT_MAPPING_IPV4 test_cilium_snat_v4_external
 #define SNAT_MAPPING_IPV4_SIZE 524288
+#define NODEPORT_NEIGH4_SIZE 524288
 #endif /* ENABLE_NODEPORT */
 #endif /* ENABLE_IPV4 */
 
 #ifdef ENABLE_IPV6
 #ifdef ENABLE_NODEPORT
 #define SNAT_MAPPING_IPV6 test_cilium_snat_v6_external
 #define SNAT_MAPPING_IPV6_SIZE 524288
+#define NODEPORT_NEIGH6_SIZE 524288
 #endif /* ENABLE_NODEPORT */
 #endif /* ENABLE_IPV6 */
 

daemon/cmd/daemon_main.go

@@ -693,6 +693,9 @@ func init() {
 	flags.Int(option.NATMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF NAT table")
 	option.BindEnv(option.NATMapEntriesGlobalName)
 
+	flags.Int(option.NeighMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF neighbor table")
+	option.BindEnv(option.NeighMapEntriesGlobalName)
+
 	flags.Int(option.PolicyMapEntriesName, defaults.PolicyMapEntries, "Maximum number of entries in endpoint policy map (per endpoint)")
 	option.BindEnv(option.PolicyMapEntriesName)
 

install/kubernetes/cilium/charts/config/templates/configmap.yaml

@@ -165,10 +165,14 @@ data:
   bpf-ct-global-tcp-max: "{{ .Values.global.bpf.ctTcpMax }}"
   bpf-ct-global-any-max: "{{ .Values.global.bpf.ctAnyMax }}"
 
-  # bpf-nat-global-max specified the maximum number of entries in the BPF NAT
-  # table.
+  # bpf-nat-global-max specified the maximum number of entries in the
+  # BPF NAT table.
   bpf-nat-global-max: "{{ .Values.global.bpf.natMax }}"
 
+  # bpf-neigh-global-max specified the maximum number of entries in the
+  # BPF neighbor table.
+  bpf-neigh-global-max: "{{ .Values.global.bpf.neighMax }}"
+
   # bpf-policy-map-max specified the maximum number of entries in endpoint
   # policy map (per endpoint)
   bpf-policy-map-max: "{{ .Values.global.bpf.policyMapMax }}"

install/kubernetes/cilium/values.yaml

@@ -242,6 +242,9 @@ global:
     # natMax is the maximum number of entries for the NAT table
     natMax: 524288
 
+    # neighMax is the maximum number of entries for the neighbor table
+    neighMax: 524288
+
     # policyMapMax is the maximum number of entries in endpoint policy map (per endpoint)
     policyMapMax: 16384
 

install/kubernetes/quick-install.yaml

@@ -87,10 +87,14 @@ data:
   bpf-ct-global-tcp-max: "524288"
   bpf-ct-global-any-max: "262144"
 
-  # bpf-nat-global-max specified the maximum number of entries in the BPF NAT
-  # table.
+  # bpf-nat-global-max specified the maximum number of entries in the
+  # BPF NAT table.
   bpf-nat-global-max: "524288"
 
+  # bpf-neigh-global-max specified the maximum number of entries in the
+  # BPF neighbor table.
+  bpf-neigh-global-max: "524288"
+
   # bpf-policy-map-max specified the maximum number of entries in endpoint
   # policy map (per endpoint)
   bpf-policy-map-max: "16384"

pkg/datapath/linux/config/config.go

@@ -239,9 +239,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC
 
 		if option.Config.EnableIPv4 {
 			cDefinesMap["NODEPORT_NEIGH4"] = neighborsmap.Map4Name
+			cDefinesMap["NODEPORT_NEIGH4_SIZE"] = fmt.Sprintf("%d", option.Config.NeighMapEntriesGlobal)
 		}
 		if option.Config.EnableIPv6 {
 			cDefinesMap["NODEPORT_NEIGH6"] = neighborsmap.Map6Name
+			cDefinesMap["NODEPORT_NEIGH6_SIZE"] = fmt.Sprintf("%d", option.Config.NeighMapEntriesGlobal)
 		}
 		if option.Config.NodePortMode == option.NodePortModeDSR ||
 			option.Config.NodePortMode == option.NodePortModeHybrid {

pkg/maps/neighborsmap/neighborsmap.go

@@ -45,7 +45,7 @@ func neighMapsGet() (*bpf.Map, *bpf.Map) {
 			int(unsafe.Sizeof(Key4{})),
 			&Value{},
 			int(unsafe.Sizeof(Value{})),
-			option.Config.NATMapEntriesGlobal,
+			option.Config.NeighMapEntriesGlobal,
 			0,
 			0,
 			bpf.ConvertKeyValue,
@@ -56,7 +56,7 @@ func neighMapsGet() (*bpf.Map, *bpf.Map) {
 			int(unsafe.Sizeof(Key6{})),
 			&Value{},
 			int(unsafe.Sizeof(Value{})),
-			option.Config.NATMapEntriesGlobal,
+			option.Config.NeighMapEntriesGlobal,
 			0,
 			0,
 			bpf.ConvertKeyValue,

pkg/option/config.go

@@ -490,6 +490,9 @@ const (
 	// NATMapEntriesGlobalName configures max entries for BPF NAT table
 	NATMapEntriesGlobalName = "bpf-nat-global-max"
 
+	// NeighMapEntriesGlobalName configures max entries for BPF neighbor table
+	NeighMapEntriesGlobalName = "bpf-neigh-global-max"
+
 	// PolicyMapEntriesName configures max entries for BPF policymap.
 	PolicyMapEntriesName = "bpf-policy-map-max"
 
@@ -789,6 +792,7 @@ var HelpFlagSections = []FlagsSection{
 			CTMapEntriesTimeoutSVCTCPName,
 			CTMapEntriesTimeoutSVCAnyName,
 			NATMapEntriesGlobalName,
+			NeighMapEntriesGlobalName,
 			PolicyMapEntriesName,
 			MapEntriesGlobalDynamicSizeRatioName,
 			PreAllocateMapsName,
@@ -1303,6 +1307,10 @@ type DaemonConfig struct {
 	// in the BPF NAT table
 	NATMapEntriesGlobal int
 
+	// NeighMapEntriesGlobal is the maximum number of neighbor mappings
+	// allowed in the BPF neigh table
+	NeighMapEntriesGlobal int
+
 	// PolicyMapEntries is the maximum number of peer identities that an
 	// endpoint may allow traffic to exchange traffic with.
 	PolicyMapEntries int
@@ -2566,6 +2574,7 @@ func (c *DaemonConfig) calculateBPFMapSizes() error {
 	c.CTMapEntriesGlobalTCP = viper.GetInt(CTMapEntriesGlobalTCPName)
 	c.CTMapEntriesGlobalAny = viper.GetInt(CTMapEntriesGlobalAnyName)
 	c.NATMapEntriesGlobal = viper.GetInt(NATMapEntriesGlobalName)
+	c.NeighMapEntriesGlobal = viper.GetInt(NeighMapEntriesGlobalName)
 	c.PolicyMapEntries = viper.GetInt(PolicyMapEntriesName)
 
 	// Don't attempt dynamic sizing if any of the sizeof members was not
@@ -2661,6 +2670,15 @@ func (c *DaemonConfig) calculateDynamicBPFMapSizes(totalMemory uint64, dynamicSi
 	} else {
 		log.Debugf("option %s set by user to %v", NATMapEntriesGlobalName, c.NATMapEntriesGlobal)
 	}
+	if !viper.IsSet(NeighMapEntriesGlobalName) {
+		// By default we auto-size it to the same value as the NAT map since we
+		// need to keep at least as many neigh entries.
+		c.NeighMapEntriesGlobal = c.NATMapEntriesGlobal
+		log.Debugf("option %s set by dynamic sizing to %v (default %v)",
+			NeighMapEntriesGlobalName, c.NeighMapEntriesGlobal, NATMapEntriesGlobalDefault)
+	} else {
+		log.Debugf("option %s set by user to %v", NeighMapEntriesGlobalName, c.NeighMapEntriesGlobal)
+	}
 	if !viper.IsSet(PolicyMapEntriesName) {
 		c.PolicyMapEntries =
 			getEntries(defaults.PolicyMapEntries, PolicyMapMin, PolicyMapMax)