UPSTREAM: <carry>: implement "local-with-fallback" external traffic p…

…olicy

If a service has a
"traffic-policy.network.alpha.openshift.io/local-with-fallback"
annotation, then only treat it as "externalTrafficPolicy: Local" when
there are actually running local pods.

That is, if we receive traffic for such a service after the last local
pod terminates, then forward it to a remote pod rather than dropping
it.

pkg/proxy/iptables/proxier.go

@@ -121,8 +121,12 @@ type serviceInfo struct {
 	servicePortChainName     utiliptables.Chain
 	serviceFirewallChainName utiliptables.Chain
 	serviceLBChainName       utiliptables.Chain
+
+	localWithFallback bool
 }
 
+const localWithFallbackAnnotation = "traffic-policy.network.alpha.openshift.io/local-with-fallback"
+
 // returns a new proxy.ServicePort which abstracts a serviceInfo
 func newServiceInfo(port *v1.ServicePort, service *v1.Service, baseInfo *proxy.BaseServiceInfo) proxy.ServicePort {
 	info := &serviceInfo{BaseServiceInfo: baseInfo}
@@ -136,6 +140,14 @@ func newServiceInfo(port *v1.ServicePort, service *v1.Service, baseInfo *proxy.B
 	info.serviceFirewallChainName = serviceFirewallChainName(info.serviceNameString, protocol)
 	info.serviceLBChainName = serviceLBChainName(info.serviceNameString, protocol)
 
+	if _, set := service.Annotations[localWithFallbackAnnotation]; set {
+		if info.NodeLocalExternal() {
+			info.localWithFallback = true
+		} else {
+			klog.Warningf("Ignoring annotation %q on Service %s which does not have Local ExternalTrafficPolicy", localWithFallbackAnnotation, svcName)
+		}
+	}
+
 	return info
 }
 
@@ -1418,6 +1430,20 @@ func (proxier *Proxier) syncProxyRules() {
 			"-m", "addrtype", "--src-type", "LOCAL", "-j", string(svcChain))
 
 		numLocalEndpoints := len(localEndpointChains)
+
+		// If "local-with-fallback" is in effect and there are no local endpoints,
+		// then NAT the traffic and forward to a remote endpoint
+		if numLocalEndpoints == 0 && svcInfo.localWithFallback {
+			proxier.natRules.Write(
+				"-A", string(svcXlbChain),
+				"-m", "comment", "--comment", `"local-with-fallback NAT"`,
+				"-j", string(KubeMarkMasqChain),
+			)
+
+			localEndpointChains = readyEndpointChains
+			numLocalEndpoints = len(localEndpointChains)
+		}
+
 		if numLocalEndpoints == 0 {
 			// Blackhole all traffic since there are no local endpoints
 			args = append(args[:0],