Skip to content

Commit

Permalink
Preserve client IP if the selected Endpoint is local regardless of Ex…
Browse files Browse the repository at this point in the history 
…ternalTrafficPolicy

When an external client accesses to a NodePort / LoadBalancer Service on a K8s
Node, if the selected Endpoint is just on the K8s Node, then don't SNAT the
connection even the externalTrafficPolicy of the Service is Cluster.

Signed-off-by: Hongliang Liu <lhongliang@vmware.com>
  • Loading branch information
@hongliangl
hongliangl committed Apr 13, 2022
1 parent 7fea8d3 commit d4fe443
Show file tree
Hide file tree
Showing 5 changed files with 45 additions and 27 deletions.
1 change: 1 addition & 0 deletions pkg/agent/openflow/client.go
View file
Expand Up @@ -714,6 +714,7 @@ func (c *client) generatePipelines() {
c.featureService = newFeatureService(c.cookieAllocator,
c.ipProtocols,
c.nodeConfig,
c.networkConfig,
c.serviceConfig,
c.bridge,
c.enableProxy,
Expand Down
6 changes: 1 addition & 5 deletions pkg/agent/openflow/fields.go
View file
Expand Up @@ -49,11 +49,7 @@ var (
PktDestinationField = binding.NewRegField(0, 4, 7, "PacketDestination")
ToTunnelRegMark = binding.NewRegMark(PktDestinationField, tunnelVal)
ToGatewayRegMark = binding.NewRegMark(PktDestinationField, gatewayVal)
// reg0[0..7]: Union field of the packet source and destination. It is used to mark hairpin packets. Marks in this
// field include:
// - 0x11: the packet sourced from Antrea gateway interface, and destined for local Node via Antrea gateway interface.
PktUnionField = binding.NewRegField(0, 0, 7, "PacketUnion")
GatewayHairpinRegMark = binding.NewRegMark(PktUnionField, (ToGatewayRegMark.GetValue()<<ToGatewayRegMark.GetField().GetRange().Offset())|FromGatewayRegMark.GetValue())
ToUplinkRegMark = binding.NewRegMark(PktDestinationField, uplinkVal)
// reg0[8]: Mark to indicate the ofPort number of an interface is found.
OFPortFoundRegMark = binding.NewOneBitRegMark(0, 8, "OFPortFound")
// reg0[9]: Field to indicate whether the packet's source / destination MAC address needs to be rewritten.
Expand Down
56 changes: 37 additions & 19 deletions pkg/agent/openflow/pipeline.go
View file
Expand Up @@ -1394,6 +1394,7 @@ func (f *featurePodConnectivity) l3FwdFlowToRemoteViaUplink(remoteGatewayMAC net
MatchDstIPNet(peerSubnet).
Action().SetSrcMAC(f.nodeConfig.UplinkNetConfig.MAC).
Action().SetDstMAC(remoteGatewayMAC).
Action().LoadRegMark(ToUplinkRegMark).
Action().GotoTable(L3DecTTLTable.GetID()).
Done()
}
Expand All @@ -1407,6 +1408,7 @@ func (f *featurePodConnectivity) l3FwdFlowToRemoteViaUplink(remoteGatewayMAC net
MatchRegMark(AntreaFlexibleIPAMRegMark).
MatchDstIPNet(peerSubnet).
Action().SetDstMAC(remoteGatewayMAC).
Action().LoadRegMark(ToUplinkRegMark).
Action().GotoTable(L3DecTTLTable.GetID()).
Done()
}
Expand Down Expand Up @@ -2160,6 +2162,12 @@ func (f *featureNetworkPolicy) ingressClassifierFlows() []binding.Flow {
MatchRegMark(ToTunnelRegMark).
Action().GotoTable(IngressMetricTable.GetID()).
Done(),
// This generates the flow to match the packets to uplink and forward them to IngressMetricTable.
IngressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal).
Cookie(cookieID).
MatchRegMark(ToUplinkRegMark).
Action().GotoTable(IngressMetricTable.GetID()).
Done(),
}
}

Expand Down Expand Up @@ -2915,35 +2923,45 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow {
cookieID := f.cookieAllocator.Request(f.category).Raw()
var flows []binding.Flow
for _, ipProtocol := range f.ipProtocols {
flows = append(flows,
// This generates the flow to match the first packet of hairpin connection initiated through the Antrea gateway.
// ConnSNATCTMark and HairpinCTMark will be loaded in DNAT CT zone.
ServiceMarkTable.ofTable.BuildFlow(priorityNormal).
Cookie(cookieID).
MatchProtocol(ipProtocol).
MatchCTStateNew(true).
MatchCTStateTrk(true).
MatchRegMark(GatewayHairpinRegMark).
Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField).
LoadToCtMark(ConnSNATCTMark).
LoadToCtMark(HairpinCTMark).
CTDone().
Done(),
// This generates the flow to match the first packet of hairpin connection initiated through the Antrea gateway.
// ConnSNATCTMark and HairpinCTMark will be loaded in DNAT CT zone.
flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityNormal).
Cookie(cookieID).
MatchProtocol(ipProtocol).
MatchCTStateNew(true).
MatchCTStateTrk(true).
MatchRegMark(FromGatewayRegMark).
MatchRegMark(ToGatewayRegMark).
Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField).
LoadToCtMark(ConnSNATCTMark).
LoadToCtMark(HairpinCTMark).
CTDone().
Done())

var pktDstRegMarks []*binding.RegMark
if f.networkConfig.TrafficEncapMode.SupportsEncap() {
pktDstRegMarks = append(pktDstRegMarks, ToTunnelRegMark)
}
if f.networkConfig.TrafficEncapMode.SupportsNoEncap() && runtime.IsWindowsPlatform() {
pktDstRegMarks = append(pktDstRegMarks, ToUplinkRegMark)
}
for _, pktDstRegMark := range pktDstRegMarks {
// This generates the flow to match the first packet of NodePort / LoadBalancer connection initiated through the
// Antrea gateway and externalTrafficPolicy of the Service is Cluster. ConnSNATCTMark will be loaded in DNAT
// CT zone.
ServiceMarkTable.ofTable.BuildFlow(priorityLow).
// Antrea gateway and externalTrafficPolicy of the Service is Cluster, and the selected Endpoint is on remote
// Node, then ConnSNATCTMark will be loaded in DNAT CT zone which indicate the connection requires SNAT.
flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityLow).
Cookie(cookieID).
MatchProtocol(ipProtocol).
MatchCTStateNew(true).
MatchCTStateTrk(true).
MatchRegMark(FromGatewayRegMark).
MatchRegMark(pktDstRegMark).
MatchRegMark(ToClusterServiceRegMark).
Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField).
LoadToCtMark(ConnSNATCTMark).
CTDone().
Done(),
)
Done())
}
}

return flows
Expand Down
3 changes: 3 additions & 0 deletions pkg/agent/openflow/service.go
View file
Expand Up @@ -40,6 +40,7 @@ type featureService struct {
gatewayMAC net.HardwareAddr
nodePortAddresses map[binding.Protocol][]net.IP
serviceCIDRs map[binding.Protocol]net.IPNet
networkConfig *config.NetworkConfig

enableProxy bool
proxyAll bool
Expand All @@ -57,6 +58,7 @@ func newFeatureService(
cookieAllocator cookie.Allocator,
ipProtocols []binding.Protocol,
nodeConfig *config.NodeConfig,
networkConfig *config.NetworkConfig,
serviceConfig *config.ServiceConfig,
bridge binding.Bridge,
enableProxy,
Expand Down Expand Up @@ -103,6 +105,7 @@ func newFeatureService(
nodePortAddresses: nodePortAddresses,
serviceCIDRs: serviceCIDRs,
gatewayMAC: nodeConfig.GatewayConfig.MAC,
networkConfig: networkConfig,
enableProxy: enableProxy,
proxyAll: proxyAll,
connectUplinkToBridge: connectUplinkToBridge,
Expand Down
6 changes: 3 additions & 3 deletions test/integration/agent/openflow_test.go
View file
Expand Up @@ -1303,7 +1303,7 @@ func prepareNodeFlows(peerSubnet net.IPNet, peerGwIP, peerNodeIP net.IP, vMAC, l
[]*ofTestUtils.ExpectFlow{
{
MatchStr: fmt.Sprintf("priority=200,%s,reg4=0x100000/0x100000,reg8=0/0xfff,%s=%s", ipProtoStr, nwDstFieldName, peerSubnet.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_dst,goto_table:L3DecTTL", peerNodeMAC.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_dst,load:0x4->NXM_NX_REG0[4..7],goto_table:L3DecTTL", peerNodeMAC.String()),
},
},
})
Expand Down Expand Up @@ -1416,7 +1416,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
)
tableServiceMarkFlows.flows = append(tableServiceMarkFlows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x22/0xff", ActStr: fmt.Sprintf("ct(commit,table=SNATConntrackCommit,zone=%s,exec(load:0x1->NXM_NX_CT_MARK[5],load:0x1->NXM_NX_CT_MARK[6]))", ctZone)},
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ip,reg0=0x2/0xf,reg4=0x200000/0x200000", ActStr: fmt.Sprintf("ct(commit,table=SNATConntrackCommit,zone=%s,exec(load:0x1->NXM_NX_CT_MARK[5]))", ctZone)},
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ip,reg0=0x12/0xff,reg4=0x200000/0x200000", ActStr: fmt.Sprintf("ct(commit,table=SNATConntrackCommit,zone=%s,exec(load:0x1->NXM_NX_CT_MARK[5]))", ctZone)},
)
tableL3DecTTLFlows.flows = append(tableL3DecTTLFlows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=210,ip,reg0=0x2/0xf", ActStr: "goto_table:ServiceMark"},
Expand Down Expand Up @@ -1461,7 +1461,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
)
tableServiceMarkFlows.flows = append(tableServiceMarkFlows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ipv6,reg0=0x22/0xff", ActStr: "ct(commit,table=SNATConntrackCommit,zone=65510,exec(load:0x1->NXM_NX_CT_MARK[5],load:0x1->NXM_NX_CT_MARK[6]))"},
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ipv6,reg0=0x2/0xf,reg4=0x200000/0x200000", ActStr: "ct(commit,table=SNATConntrackCommit,zone=65510,exec(load:0x1->NXM_NX_CT_MARK[5]))"},
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ipv6,reg0=0x12/0xff,reg4=0x200000/0x200000", ActStr: "ct(commit,table=SNATConntrackCommit,zone=65510,exec(load:0x1->NXM_NX_CT_MARK[5]))"},
)
tableL3DecTTLFlows.flows = append(tableL3DecTTLFlows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=210,ipv6,reg0=0x2/0xf", ActStr: "goto_table:ServiceMark"},
Expand Down

0 comments on commit d4fe443

Please sign in to comment.