datapath/iptables: Masquerade hairpin traffic that traversed the stack

The traffic is sent to the stack and hairpin'ed back into a local pod
after a component on the stack has applied a DNAT rule, the traffic must
be SNATed to ensure the reverse NAT can take place. This can happen if
portmap or kiam is being used and redirection happens to a local
destination.

The masquerade filter must be limited as not all DNAT traffic may be
affected. NodePort traffic from a non-local source must remain
unmasqueraded in order for trafficPolicy=local to continue working.

Also, when EnableEndpointRoutes is enabled, traffic always traverses the
stack and must not be masqueraded either.

Fixes: #9784

Signed-off-by: Thomas Graf <thomas@cilium.io>

pkg/datapath/iptables/iptables.go

@@ -959,6 +959,35 @@ func (m *IptablesManager) InstallRules(ifName string) error {
 				"-j", "SNAT", "--to-source", node.GetHostMasqueradeIPv4().String()), false); err != nil {
 				return err
 			}
+
+			// Masquerade all traffic that originated from a local
+			// pod and thus carries a security identity and that
+			// was also DNAT'ed. It must be masqueraded to ensure
+			// that reverse NAT can be performed. Otherwise the
+			// reply traffic would be sent directly to the pod
+			// without traversing the Linux stack again.
+			//
+			// This is only done if EnableEndpointRoutes is
+			// disabled, if EnableEndpointRoutes is enabled, then
+			// all traffic always passes through the stack anyway.
+			//
+			// This is required for:
+			//  - portmap/host if both source and destination are
+			//    on the same node
+			//  - kiam if source and server are on the same node
+			if !option.Config.EnableEndpointRoutes {
+				if err := runProg("iptables", append(
+					m.waitArgs,
+					"-t", "nat",
+					"-A", ciliumPostNatChain,
+					"-m", "mark", "--mark", fmt.Sprintf("%#08x/%#08x", linux_defaults.MagicMarkIdentity, linux_defaults.MagicMarkHostMask),
+					"-o", localDeliveryInterface,
+					"-m", "conntrack", "--ctstate", "DNAT",
+					"-m", "comment", "--comment", "hairpin traffic that originated from a local pod",
+					"-j", "SNAT", "--to-source", node.GetHostMasqueradeIPv4().String()), false); err != nil {
+					return err
+				}
+			}
 		}
 	}
 

pkg/datapath/linux/linux_defaults/mark.go

@@ -67,6 +67,11 @@ const (
 	// MagicMarkHost determines that the traffic is sourced from the local
 	// host and not from a proxy.
 	MagicMarkHost int = 0x0C00
+
+	// MagicMarkIdentity determines that the traffic carries a security
+	// identity in the skb->mark
+	MagicMarkIdentity int = 0x0F00
+
 	// MagicMarkK8sMasq determines that the traffic should be masqueraded
 	// by kube-proxy in kubernetes environments.
 	MagicMarkK8sMasq int = 0x4000