bpf: derive identity from ipcache for external traffic under fast red…

…irect

We're in a similar situation as ipvlan datapath here in that we must
derive the secid for policy enforcement via SECCTX_FROM_IPCACHE. This
is needed as now we do not push the packet up the stack anymore where
it will take the tc egress path of the bpf_lxc dev where it would resolve
the secid, but instead we pass it onwards via ipv{4,6}_local_delivery()
from bpf_host given we do not have skip_redirect. So in the latter this
gets encoded via CB_SRC_LABEL before tail calling into ep->lxc_id. In
bpf_host resolve_srcid_ipv4() was always picking WORLD (2) which will
fail CIDR-based enforcement, e.g. hubble logs revealed this:

{"time":"2020-11-13T13:53:09.636444980Z","verdict":"DROPPED","drop_reason":133,"ethernet":{"source":"0a:4b:c4:b6:2d:4b","destination":"92:79:4f:8e:96:4f"},"IP":{"source":"192.168.36.13","destination":"10.0.1.190","ipVersion":"IPv4"},"l4":{"TCP":{"source_port":56228,"destination_port":80,"flags":{"SYN":true}}},"source":{"identity":2,"labels":["reserved:world"]},"destination":{"ID":1091,"identity":41849,"namespace":"default","labels":["k8s:io.cilium.k8s.policy.cluster=default","k8s:io.cilium.k8s.policy.serviceaccount=default","k8s:io.kubernetes.pod.namespace=default","k8s:zgroup=testDS"],"pod_name":"testds-944zc"},"Type":"L3_L4","node_name":"k8s2","event_type":{"type":5},"traffic_direction":"INGRESS","drop_reason_desc":"POLICY_DENIED","Summary":"TCP Flags: SYN"}

The source identity in this case should have been 16777217 as per ipcache
dump of ...

  [...]
  192.168.36.13/32               16777217   0   0.0.0.0
  [...]

... and thus it failed CI test 'Suite-k8s-1.12.K8sPolicyTest Multi-node policy
test validates ingress CIDR-dependent L4 connectivity is restored after
importing ingress policy' where a non-Cilium managed node (192.168.36.13) made
a request to the backend Pod directly via curl. It succeeded before policy
to allow 192.168.36.13 to port 80 was installed but failed after the latter
was set in place due to secid mismatch. Fix it by enabling SECCTX_FROM_IPCACHE
so that it can assign correct identity in resolve_srcid_ipv{4,6}().

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

bpf/bpf_host.c

@@ -66,6 +66,16 @@ static __always_inline int rewrite_dmac_to_host(struct __ctx_buff *ctx,
 
 	return CTX_ACT_OK;
 }
+
+#define SECCTX_FROM_IPCACHE_OK	2
+#ifndef SECCTX_FROM_IPCACHE
+# define SECCTX_FROM_IPCACHE	0
+#endif
+
+static __always_inline bool identity_from_ipcache_ok(void)
+{
+	return SECCTX_FROM_IPCACHE == SECCTX_FROM_IPCACHE_OK;
+}
 #endif
 
 #ifdef ENABLE_IPV6
@@ -144,15 +154,10 @@ resolve_srcid_ipv6(struct __ctx_buff *ctx, __u32 srcid_from_proxy,
 
 	if (from_host)
 		src_id = srcid_from_ipcache;
-#ifdef ENABLE_SECCTX_FROM_IPCACHE
-	/* If we could not derive the secctx from the packet itself but
-	 * from the ipcache instead, then use the ipcache identity. E.g.
-	 * used in ipvlan master device's datapath on ingress.
-	 */
-	else if (src_id == WORLD_ID && !identity_is_reserved(srcid_from_ipcache))
+	else if (src_id == WORLD_ID &&
+		 identity_from_ipcache_ok() &&
+		 !identity_is_reserved(srcid_from_ipcache))
 		src_id = srcid_from_ipcache;
-#endif
-
 	return src_id;
 }
 
@@ -410,15 +415,12 @@ resolve_srcid_ipv4(struct __ctx_buff *ctx, __u32 srcid_from_proxy,
 
 	if (from_host)
 		src_id = srcid_from_ipcache;
-#ifdef ENABLE_SECCTX_FROM_IPCACHE
 	/* If we could not derive the secctx from the packet itself but
-	 * from the ipcache instead, then use the ipcache identity. E.g.
-	 * used in ipvlan master device's datapath on ingress.
+	 * from the ipcache instead, then use the ipcache identity.
 	 */
-	else if (!identity_is_reserved(srcid_from_ipcache))
+	else if (identity_from_ipcache_ok() &&
+		 !identity_is_reserved(srcid_from_ipcache))
 		src_id = srcid_from_ipcache;
-#endif
-
 	return src_id;
 }
 

pkg/datapath/linux/config/config.go

@@ -156,7 +156,6 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC
 	cDefinesMap["CT_REPORT_FLAGS"] = fmt.Sprintf("%#04x", int64(option.Config.MonitorAggregationFlags))
 
 	if option.Config.DatapathMode == datapathOption.DatapathModeIpvlan {
-		cDefinesMap["ENABLE_SECCTX_FROM_IPCACHE"] = "1"
 		cDefinesMap["ENABLE_EXTRA_HOST_DEV"] = "1"
 	}
 
@@ -515,7 +514,8 @@ func (h *HeaderfileWriter) writeStaticData(fw io.Writer, e datapath.EndpointConf
 			placeholderIPv4 := []byte{1, 1, 1, 1}
 			fmt.Fprint(fw, defineIPv4("IPV4_MASQUERADE", placeholderIPv4))
 		}
-
+		// Dummy value to avoid being optimized when 0
+		fmt.Fprint(fw, defineUint32("SECCTX_FROM_IPCACHE", 1))
 		fmt.Fprint(fw, defineUint32("HOST_EP_ID", uint32(e.GetID())))
 	} else {
 		// We want to ensure that the template BPF program always has "LXC_IP"

pkg/datapath/loader/loader.go

@@ -57,6 +57,11 @@ const (
 	dirEgress  = "egress"
 )
 
+const (
+	SecctxFromIpcacheDisabled = iota + 1
+	SecctxFromIpcacheEnabled
+)
+
 var log = logging.DefaultLogger.WithField(logfields.LogSubsys, Subsystem)
 
 // Loader is a wrapper structure around operations related to compiling,
@@ -156,6 +161,13 @@ func patchHostNetdevDatapath(ep datapath.Endpoint, objPath, dstPath, ifName stri
 		return err
 	}
 
+	if !option.Config.EnableHostLegacyRouting ||
+		option.Config.DatapathMode == datapathOption.DatapathModeIpvlan {
+		opts["SECCTX_FROM_IPCACHE"] = uint32(SecctxFromIpcacheEnabled)
+	} else {
+		opts["SECCTX_FROM_IPCACHE"] = uint32(SecctxFromIpcacheDisabled)
+	}
+
 	if option.Config.EnableNodePort && nodePortIPv4Addrs != nil && nodePortIPv6Addrs != nil {
 		opts["NATIVE_DEV_IFINDEX"] = ifIndex
 		if option.Config.EnableIPv4 {

pkg/datapath/loader/template.go

@@ -246,6 +246,7 @@ func elfVariableSubstitutions(ep datapath.Endpoint) map[string]uint32 {
 				result["IPV4_MASQUERADE"] = 0
 			}
 		}
+		result["SECCTX_FROM_IPCACHE"] = uint32(SecctxFromIpcacheDisabled)
 		result["HOST_EP_ID"] = uint32(ep.GetID())
 	} else {
 		result["LXC_ID"] = uint32(ep.GetID())