datapath: disable net.ipv4.ip_early_demux for IPsec + L7 proxy

[ upstream commit 5201896 ]

After forward traffic for an egress proxy onnection has traversed through
cilium_host / cilium_net, we expect IPsec-marked packets to get handled
by xfrm.

This currently conflicts with early demux, which matches the connection's
transparent socket and assigns it to the packet:

```
// https://elixir.bootlin.com/linux/v6.2/source/net/ipv4/tcp_ipv4.c#L1770
int tcp_v4_early_demux(struct sk_buff *skb)
{
...
	sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
				       iph->saddr, th->source,
				       iph->daddr, ntohs(th->dest),
				       skb->skb_iif, inet_sdif(skb));
	if (sk) {
		skb->sk = sk;
...
}
```

It then gets dropped in ip_forward(), before reaching xfrm:

```
// https://elixir.bootlin.com/linux/v6.2/source/net/ipv4/ip_forward.c#L100
int ip_forward(struct sk_buff *skb)
{
...
    if (unlikely(skb->sk))
		goto drop;
...
}
```

To avoid this we disable early-demux in a L7 + IPsec config.

Note that the L7 proxy feature needs to deal with similar troubles, as the
comment for inboundProxyRedirectRule() describes. Ideally we would build
a similar solution for IPsec, diverting traffic with policy routing so that
it doesn't get intercepted by early-demux.

Signed-off-by: Zhichuan Liang<gray.liang@isovalent.com>
Signed-off-by: Julian Wiedmann <jwi@isovalent.com>

pkg/datapath/iptables/cell.go

@@ -38,6 +38,7 @@ var Cell = cell.Module(
 			EnableIPSec:                 cfg.EnableIPSec,
 			MasqueradeInterfaces:        cfg.MasqueradeInterfaces,
 			EnableMasqueradeRouteSource: cfg.EnableMasqueradeRouteSource,
+			EnableL7Proxy:               cfg.EnableL7Proxy,
 		}
 	}),
 	cell.Provide(newIptablesManager),
@@ -90,4 +91,5 @@ type SharedConfig struct {
 	EnableIPSec                 bool
 	MasqueradeInterfaces        []string
 	EnableMasqueradeRouteSource bool
+	EnableL7Proxy               bool
 }

pkg/datapath/iptables/iptables.go

@@ -309,6 +309,10 @@ func (m *Manager) Start(ctx cell.HookContext) error {
 		m.logger.WithError(err).Warning("enabling IP forwarding via sysctl failed")
 	}
 
+	if m.sharedCfg.EnableIPSec && m.sharedCfg.EnableL7Proxy {
+		m.DisableIPEarlyDemux()
+	}
+
 	if err := m.modulesMgr.FindOrLoadModules(
 		"ip_tables", "iptable_nat", "iptable_mangle", "iptable_raw", "iptable_filter",
 	); err != nil {
@@ -364,14 +368,7 @@ func (m *Manager) Start(ctx cell.HookContext) error {
 			m.logger.WithError(err).Warning("xt_socket kernel module could not be loaded")
 
 			if m.sharedCfg.EnableXTSocketFallback {
-				disabled := sysctl.Disable("net.ipv4.ip_early_demux") == nil
-
-				if disabled {
-					m.ipEarlyDemuxDisabled = true
-					m.logger.Warning("Disabled ip_early_demux to allow proxy redirection with original source/destination address without xt_socket support also in non-tunneled datapath modes.")
-				} else {
-					m.logger.WithError(err).Warning("Could not disable ip_early_demux, traffic redirected due to an HTTP policy or visibility may be dropped unexpectedly")
-				}
+				m.DisableIPEarlyDemux()
 			}
 		}
 	} else {
@@ -389,6 +386,20 @@ func (m *Manager) Stop(ctx cell.HookContext) error {
 	return nil
 }
 
+func (m *Manager) DisableIPEarlyDemux() {
+	if m.ipEarlyDemuxDisabled {
+		return
+	}
+
+	err := sysctl.Disable("net.ipv4.ip_early_demux")
+	if err == nil {
+		m.ipEarlyDemuxDisabled = true
+		m.logger.Info("Disabled ip_early_demux to allow proxy redirection.")
+	} else {
+		m.logger.WithError(err).Warning("Could not disable ip_early_demux, traffic redirected due to an HTTP policy or visibility may be dropped unexpectedly")
+	}
+}
+
 // SupportsOriginalSourceAddr tells if an L7 proxy can use POD's original source address and port in
 // the upstream connection to allow the destination to properly derive the source security ID from
 // the source IP address.