From 779e50402081080ecc5d95bc1c8ec95233d4076b Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Tue, 2 Apr 2024 20:53:56 +0800 Subject: [PATCH] Support Service loadBalancerSourceRanges in AntreaProxy For #5493 Signed-off-by: Hongliang Liu --- docs/design/ovs-pipeline.md | 220 ++++++++++++++++--------- pkg/agent/openflow/client.go | 3 + pkg/agent/openflow/client_test.go | 96 +++++++---- pkg/agent/openflow/fields.go | 6 +- pkg/agent/openflow/framework.go | 4 +- pkg/agent/openflow/framework_test.go | 4 +- pkg/agent/openflow/pipeline.go | 45 +++++- pkg/agent/openflow/pipeline_test.go | 6 +- pkg/agent/openflow/service.go | 2 +- pkg/agent/openflow/service_test.go | 12 +- pkg/agent/proxy/proxier.go | 41 +++-- pkg/agent/proxy/proxier_test.go | 233 +++++++++++++++++++++------ pkg/agent/types/service.go | 3 +- 13 files changed, 473 insertions(+), 202 deletions(-) diff --git a/docs/design/ovs-pipeline.md b/docs/design/ovs-pipeline.md index 6f5d1d18b24..33fdb8c0ae5 100644 --- a/docs/design/ovs-pipeline.md +++ b/docs/design/ovs-pipeline.md @@ -108,63 +108,64 @@ where `` is the name of a table in the pipeline, and `` is We use some OVS registers to carry information throughout the pipeline. To enhance usability, we assign friendly names to the registers we use. -| Register | Field Range | Field Name | Reg Mark Value | Reg Mark Name | Description | -|---------------|-------------|---------------------------------|----------------|---------------------------------|------------------------------------------------------------------------------------------------------| -| NXM_NX_REG0 | bits 0-3 | PktSourceField | 0x1 | FromTunnelRegMark | Packet source is tunnel port. | -| | | | 0x2 | FromGatewayRegMark | Packet source is the local Antrea gateway port. | -| | | | 0x3 | FromPodRegMark | Packet source is local Pod port. | -| | | | 0x4 | FromUplinkRegMark | Packet source is uplink port. | -| | | | 0x5 | FromBridgeRegMark | Packet source is local bridge port. | -| | | | 0x6 | FromTCReturnRegMark | Packet source is TrafficControl return port. | -| | bits 4-7 | PktDestinationField | 0x1 | ToTunnelRegMark | Packet destination is tunnel port. | -| | | | 0x2 | ToGatewayRegMark | Packet destination is the local Antrea gateway port. | -| | | | 0x3 | ToLocalRegMark | Packet destination is local Pod port. | -| | | | 0x4 | ToUplinkRegMark | Packet destination is uplink port. | -| | | | 0x5 | ToBridgeRegMark | Packet destination is local bridge port. | -| | bit 9 | | 0b0 | NotRewriteMACRegMark | Packet's source/destination MAC address does not need to be rewritten. | -| | | | 0b1 | RewriteMACRegMark | Packet's source/destination MAC address needs to be rewritten. | -| | bit 10 | | 0b1 | APDenyRegMark | Packet denied (Drop/Reject) by Antrea NetworkPolicy. | -| | bits 11-12 | APDispositionField | 0b00 | DispositionAllowRegMark | Indicates Antrea NetworkPolicy disposition: allow. | -| | | | 0b01 | DispositionDropRegMark | Indicates Antrea NetworkPolicy disposition: drop. | -| | | | 0b11 | DispositionPassRegMark | Indicates Antrea NetworkPolicy disposition: pass. | -| | bit 13 | | 0b1 | GeneratedRejectPacketOutRegMark | Indicates packet is a generated reject response packet-out. | -| | bit 14 | | 0b1 | SvcNoEpRegMark | Indicates packet towards a Service without Endpoint. | -| | bit 19 | | 0b1 | RemoteSNATRegMark | Indicates packet needs SNAT on a remote Node. | -| | bit 22 | | 0b1 | L7NPRedirectRegMark | Indicates L7 Antrea NetworkPolicy disposition of redirect. | -| | bits 21-22 | OutputRegField | 0b01 | OutputToOFPortRegMark | Output packet to an OVS port. | -| | | | 0b10 | OutputToControllerRegMark | Send packet to Antrea Agent. | -| | bits 25-32 | PacketInOperationField | | | Field to store NetworkPolicy packetIn operation. | -| NXM_NX_REG1 | bits 0-31 | TargetOFPortField | | | Egress OVS port of packet. | -| NXM_NX_REG2 | bits 0-31 | SwapField | | | Swap values in flow fields in OpenFlow actions. | -| | bits 0-7 | PacketInTableField | | | OVS table where it was decided to send packets to the controller (Antrea Agent). | -| NXM_NX_REG3 | bits 0-31 | EndpointIPField | | | Field to store IPv4 address of the selected Service Endpoint. | -| | | APConjIDField | | | Field to store Conjunction ID for Antrea Policy. | -| NXM_NX_REG4 | bits 0-15 | EndpointPortField | | | Field store TCP/UDP/SCTP port of a Service's selected Endpoint. | -| | bits 16-18 | ServiceEPStateField | 0b001 | EpToSelectRegMark | Packet needs to do Service Endpoint selection. | -| | bits 16-18 | ServiceEPStateField | 0b010 | EpSelectedRegMark | Packet has done Service Endpoint selection. | -| | bits 16-18 | ServiceEPStateField | 0b011 | EpToLearnRegMark | Packet has done Service Endpoint selection and the selected Endpoint needs to be cached. | -| | bits 0-18 | EpUnionField | | | The union value of EndpointPortField and ServiceEPStateField. | -| | bit 19 | | 0b1 | ToNodePortAddressRegMark | Packet is destined for a Service of type NodePort. | -| | bit 20 | | 0b1 | AntreaFlexibleIPAMRegMark | Packet is from local Antrea IPAM Pod. | -| | bit 20 | | 0b0 | NotAntreaFlexibleIPAMRegMark | Packet is not from local Antrea IPAM Pod. | -| | bit 21 | | 0b1 | ToExternalAddressRegMark | Packet is destined for a Service's external IP. | -| | bits 22-23 | TrafficControlActionField | 0b01 | TrafficControlMirrorRegMark | Indicates packet needs to be mirrored (used by TrafficControl). | -| | | | 0b10 | TrafficControlRedirectRegMark | Indicates packet needs to be redirected (used by TrafficControl). | -| | bit 24 | | 0b1 | NestedServiceRegMark | Packet is destined for a Service using other Services as Endpoints. | -| | bit 25 | | 0b1 | DSRServiceRegMark | Packet is destined for a Service working in DSR mode. | -| | | | 0b0 | NotDSRServiceRegMark | Packet is destined for a Service working in non-DSR mode. | -| | bit 26 | | 0b1 | RemoteEndpointRegMark | Packet is destined for a Service selecting a remote non-hostNetwork Endpoint. | -| | bit 27 | | 0b1 | FromExternalRegMark | Packet is from Antrea gateway, but its source IP is not the gateway IP. | -| | bit 28 | | 0b1 | FromLocalRegMark | Packet is from a local Pod or the Node. | -| NXM_NX_REG5 | bits 0-31 | TFEgressConjIDField | | | Egress conjunction ID hit by TraceFlow packet. | -| NXM_NX_REG6 | bits 0-31 | TFIngressConjIDField | | | Ingress conjunction ID hit by TraceFlow packet. | -| NXM_NX_REG7 | bits 0-31 | ServiceGroupIDField | | | GroupID corresponding to the Service. | -| NXM_NX_REG8 | bits 0-11 | VLANIDField | | | VLAN ID. | -| | bits 12-15 | CtZoneTypeField | 0b0001 | IPCtZoneTypeRegMark | Ct zone type is IPv4. | -| | | | 0b0011 | IPv6CtZoneTypeRegMark | Ct zone type is IPv6. | -| | bits 0-15 | CtZoneField | | | Ct zone ID is a combination of VLANIDField and CtZoneTypeField. | -| NXM_NX_REG9 | bits 0-31 | TrafficControlTargetOFPortField | | | Field to cache the OVS port to output packets to be mirrored or redirected (used by TrafficControl). | -| NXM_NX_XXREG3 | bits 0-127 | EndpointIP6Field | | | Field to store IPv6 address of the selected Service Endpoint. | +| Register | Field Range | Field Name | Reg Mark Value | Reg Mark Name | Description | +|---------------|-------------|---------------------------------|----------------|---------------------------------|----------------------------------------------------------------------------------------------------------| +| NXM_NX_REG0 | bits 0-3 | PktSourceField | 0x1 | FromTunnelRegMark | Packet source is tunnel port. | +| | | | 0x2 | FromGatewayRegMark | Packet source is the local Antrea gateway port. | +| | | | 0x3 | FromPodRegMark | Packet source is local Pod port. | +| | | | 0x4 | FromUplinkRegMark | Packet source is uplink port. | +| | | | 0x5 | FromBridgeRegMark | Packet source is local bridge port. | +| | | | 0x6 | FromTCReturnRegMark | Packet source is TrafficControl return port. | +| | bits 4-7 | PktDestinationField | 0x1 | ToTunnelRegMark | Packet destination is tunnel port. | +| | | | 0x2 | ToGatewayRegMark | Packet destination is the local Antrea gateway port. | +| | | | 0x3 | ToLocalRegMark | Packet destination is local Pod port. | +| | | | 0x4 | ToUplinkRegMark | Packet destination is uplink port. | +| | | | 0x5 | ToBridgeRegMark | Packet destination is local bridge port. | +| | bit 9 | | 0b0 | NotRewriteMACRegMark | Packet's source/destination MAC address does not need to be rewritten. | +| | | | 0b1 | RewriteMACRegMark | Packet's source/destination MAC address needs to be rewritten. | +| | bit 10 | | 0b1 | APDenyRegMark | Packet denied (Drop/Reject) by Antrea NetworkPolicy. | +| | bits 11-12 | APDispositionField | 0b00 | DispositionAllowRegMark | Indicates Antrea NetworkPolicy disposition: allow. | +| | | | 0b01 | DispositionDropRegMark | Indicates Antrea NetworkPolicy disposition: drop. | +| | | | 0b11 | DispositionPassRegMark | Indicates Antrea NetworkPolicy disposition: pass. | +| | bit 13 | | 0b1 | GeneratedRejectPacketOutRegMark | Indicates packet is a generated reject response packet-out. | +| | bit 14 | | 0b1 | SvcRejectRegMark | Indicates packet towards a Service should be rejected. | +| | bit 19 | | 0b1 | RemoteSNATRegMark | Indicates packet needs SNAT on a remote Node. | +| | bit 22 | | 0b1 | L7NPRedirectRegMark | Indicates L7 Antrea NetworkPolicy disposition of redirect. | +| | bits 21-22 | OutputRegField | 0b01 | OutputToOFPortRegMark | Output packet to an OVS port. | +| | | | 0b10 | OutputToControllerRegMark | Send packet to Antrea Agent. | +| | bits 25-32 | PacketInOperationField | | | Field to store NetworkPolicy packetIn operation. | +| NXM_NX_REG1 | bits 0-31 | TargetOFPortField | | | Egress OVS port of packet. | +| NXM_NX_REG2 | bits 0-31 | SwapField | | | Swap values in flow fields in OpenFlow actions. | +| | bits 0-7 | PacketInTableField | | | OVS table where it was decided to send packets to the controller (Antrea Agent). | +| NXM_NX_REG3 | bits 0-31 | EndpointIPField | | | Field to store IPv4 address of the selected Service Endpoint. | +| | | APConjIDField | | | Field to store Conjunction ID for Antrea Policy. | +| NXM_NX_REG4 | bits 0-15 | EndpointPortField | | | Field store TCP/UDP/SCTP port of a Service's selected Endpoint. | +| | bits 16-18 | ServiceEPStateField | 0b001 | EpToSelectRegMark | Packet needs to do Service Endpoint selection. | +| | bits 16-18 | ServiceEPStateField | 0b010 | EpSelectedRegMark | Packet has done Service Endpoint selection. | +| | bits 16-18 | ServiceEPStateField | 0b011 | EpToLearnRegMark | Packet has done Service Endpoint selection and the selected Endpoint needs to be cached. | +| | bits 0-18 | EpUnionField | | | The union value of EndpointPortField and ServiceEPStateField. | +| | bit 19 | | 0b1 | ToNodePortAddressRegMark | Packet is destined for a Service of type NodePort. | +| | bit 20 | | 0b1 | AntreaFlexibleIPAMRegMark | Packet is from local Antrea IPAM Pod. | +| | bit 20 | | 0b0 | NotAntreaFlexibleIPAMRegMark | Packet is not from local Antrea IPAM Pod. | +| | bit 21 | | 0b1 | ToExternalAddressRegMark | Packet is destined for a Service's external IP. | +| | bits 22-23 | TrafficControlActionField | 0b01 | TrafficControlMirrorRegMark | Indicates packet needs to be mirrored (used by TrafficControl). | +| | | | 0b10 | TrafficControlRedirectRegMark | Indicates packet needs to be redirected (used by TrafficControl). | +| | bit 24 | | 0b1 | NestedServiceRegMark | Packet is destined for a Service using other Services as Endpoints. | +| | bit 25 | | 0b1 | DSRServiceRegMark | Packet is destined for a Service working in DSR mode. | +| | | | 0b0 | NotDSRServiceRegMark | Packet is destined for a Service working in non-DSR mode. | +| | bit 26 | | 0b1 | RemoteEndpointRegMark | Packet is destined for a Service selecting a remote non-hostNetwork Endpoint. | +| | bit 27 | | 0b1 | FromExternalRegMark | Packet is from Antrea gateway, but its source IP is not the gateway IP. | +| | bit 28 | | 0b1 | FromLocalRegMark | Packet is from a local Pod or the Node. | +| | bit 29 | | 0b1 | LoadBalancerSourceRangesRegMark | Source of packet destined for a LoadBalancer Service is included in the Service LoadBalancerSourceRanges | +| NXM_NX_REG5 | bits 0-31 | TFEgressConjIDField | | | Egress conjunction ID hit by TraceFlow packet. | +| NXM_NX_REG6 | bits 0-31 | TFIngressConjIDField | | | Ingress conjunction ID hit by TraceFlow packet. | +| NXM_NX_REG7 | bits 0-31 | ServiceGroupIDField | | | GroupID corresponding to the Service. | +| NXM_NX_REG8 | bits 0-11 | VLANIDField | | | VLAN ID. | +| | bits 12-15 | CtZoneTypeField | 0b0001 | IPCtZoneTypeRegMark | Ct zone type is IPv4. | +| | | | 0b0011 | IPv6CtZoneTypeRegMark | Ct zone type is IPv6. | +| | bits 0-15 | CtZoneField | | | Ct zone ID is a combination of VLANIDField and CtZoneTypeField. | +| NXM_NX_REG9 | bits 0-31 | TrafficControlTargetOFPortField | | | Field to cache the OVS port to output packets to be mirrored or redirected (used by TrafficControl). | +| NXM_NX_XXREG3 | bits 0-127 | EndpointIP6Field | | | Field to store IPv6 address of the selected Service Endpoint. | Note that reg marks that have overlapped bits will not be used at the same time, such as `SwapField` and `PacketInTableField`. @@ -254,7 +255,7 @@ spec: ## Kubernetes Service Implementation Like K8s NetworkPolicy, several tables of the pipeline are dedicated to [Kubernetes -Service](https://kubernetes.io/docs/concepts/services-networking/service/) implementation (tables [NodePortMark], +Service](https://kubernetes.io/docs/concepts/services-networking/service/) implementation (tables [ServiceMark], [SessionAffinity], [ServiceLB], and [EndpointDNAT]). By enabling `proxyAll`, ClusterIP, NodePort, LoadBalancer, and ExternalIP are all handled by AntreaProxy. Otherwise, @@ -408,6 +409,34 @@ status: - ip: 192.168.77.151 ``` +### LoadBalancer with LoadBalancerSourceRanges + +A sample LoadBalancer Service, with ingress IP `192.168.77.150` assigned by an ingress controller, configured +`loadBalancerSourceRanges` to a CIDR list and `externalTrafficPolicy` to `Local`. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: sample-loadbalancer +spec: + selector: + app: web + ports: + - protocol: TCP + port: 80 + targetPort: 80 + type: LoadBalancer + externalTrafficPolicy: Local + loadBalancerSourceRanges: + - "192.168.77.0/24" + - "192.168.78.0/24" +status: + loadBalancer: + ingress: + - ip: 192.168.77.152 +``` + ## Antrea-native NetworkPolicy Implementation In addition to the tables created for K8s NetworkPolicy, Antrea creates additional dedicated tables to support @@ -862,7 +891,7 @@ specific cases: 1. Dropping invalid packets reported by conntrack. 2. Forwarding tracked packets from all connections to table [AntreaPolicyEgressRule] directly, bypassing the tables - like [PreRoutingClassifier], [NodePortMark], [SessionAffinity], [ServiceLB], and [EndpointDNAT] for Service Endpoint + like [PreRoutingClassifier], [ServiceMark], [SessionAffinity], [ServiceLB], and [EndpointDNAT] for Service Endpoint selection. 3. Forwarding packets from new connections to table [PreRoutingClassifier] to start Service Endpoint selection since Service connections are not identified at this stage. @@ -892,35 +921,42 @@ Flow 4 is the table-miss flow for case 3, matching packets from all new connecti ### PreRoutingClassifier This table handles the first packet from uncommitted Service connections before Service Endpoint selection. It -sequentially resubmits the packets to tables [NodePortMark] and [SessionAffinity] to do some pre-processing, including +sequentially resubmits the packets to tables [ServiceMark] and [SessionAffinity] to do some pre-processing, including the loading of specific reg marks. Subsequently, it forwards the packets to table [ServiceLB] to perform Service Endpoint selection. If you dump the flows of this table, you may see the following: ```text -1. table=PreRoutingClassifier, priority=200,ip actions=resubmit(,NodePortMark),resubmit(,SessionAffinity),resubmit(,ServiceLB) -2. table=PreRoutingClassifier, priority=0 actions=goto_table:NodePortMark +1. table=PreRoutingClassifier, priority=200,ip actions=resubmit(,ServiceMark),resubmit(,SessionAffinity),resubmit(,ServiceLB) +2. table=PreRoutingClassifier, priority=0 actions=goto_table:ServiceMark ``` -Flow 1 sequentially resubmits packets to tables [NodePortMark], [SessionAffinity], and [ServiceLB]. Note that packets -are ultimately forwarded to table [ServiceLB]. In tables [NodePortMark] and [SessionAffinity], only reg marks are loaded. +Flow 1 sequentially resubmits packets to tables [ServiceMark], [SessionAffinity], and [ServiceLB]. Note that packets +are ultimately forwarded to table [ServiceLB]. In tables [ServiceMark] and [SessionAffinity], only reg marks are loaded. Flow 2 is the table-miss flow that should remain unused. -### NodePortMark +### ServiceMark + +This table is designed to mark some patterns of Service traffic. It addresses specific cases: -This table is designed to potentially mark packets destined for NodePort Services. It is only created when `proxyAll` is -enabled. +1. Packets potentially destined for NodePort Services. +2. Packets destined for LoadBalancer Services configured with `loadBalancerSourceRanges`. If you dump the flows of this table, you may see the following: ```text -1. table=NodePortMark, priority=200,ip,nw_dst=192.168.77.102 actions=set_field:0x80000/0x80000->reg4 -2. table=NodePortMark, priority=200,ip,nw_dst=169.254.0.252 actions=set_field:0x80000/0x80000->reg4 -3. table=NodePortMark, priority=0 actions=goto_table:SessionAffinity +1. table=ServiceMark, priority=200,ip,nw_dst=192.168.77.102 actions=set_field:0x80000/0x80000->reg4 +2. table=ServiceMark, priority=200,ip,nw_dst=169.254.0.252 actions=set_field:0x80000/0x80000->reg4 +3. table=ServiceMark, priority=200,tcp,nw_src=192.168.77.0/24,nw_dst=192.168.77.152,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", +4. table=ServiceMark, priority=200,tcp,nw_src=192.168.78.0/24,nw_dst=192.168.77.152,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", +5. table=ServiceMark, priority=190,tcp,nw_dst=192.168.77.152,tp_dst=80 actions=set_field:0x4000/0x4000->reg0", +6. table=ServiceMark, priority=0 actions=goto_table:SessionAffinity ``` +Flows 1-2 are designed for case 1. + Flow 1 matches packets destined for the local Node from local Pods. `NodePortRegMark` is loaded, indicating that the packets are potentially destined for NodePort Services. We assume only one valid IP address, `192.168.77.102` (the Node's transport IP), can serve as the host IP address for NodePort based on the option `antreaProxy.nodePortAddresses`. @@ -930,7 +966,20 @@ IP address. Flow 2 match packets destined for the *Virtual NodePort DNAT IP*. Packets destined for NodePort Services from the local Node or the external network is DNAT'd to the *Virtual NodePort DNAT IP* by iptables before entering the pipeline. -Flow 3 is the table-miss flow. +Flows 3-5 are designed for case 2. + +Flow 3-4 are used to match the packets destined for the sample [LoadBalancer with LoadBalancerSourceRanges], and these +packets are also from the CIDRs within the `loadBalancerSourceRanges` of the Services. `LoadBalancerSourceRangesRegMark`, +which will be consumed in table [ServiceLB], is loaded to identify that the corresponding connections should get +load-balanced in table [ServiceLB]. + +Flow 5, having lower priority than flow 3-4, is used to match the packets destined for the sample [LoadBalancer with +LoadBalancerSourceRanges], but these packets are not from any CIDRs within the `loadBalancerSourceRanges` of the Services. +`SvcRejectRegMark`, which will be consumed in table [EndpointDNAT], is loaded to identify that the corresponding +connections should be rejected. Since `LoadBalancerSourceRangesRegMark` is not loaded for the packets, the corresponding +connections will not get load-balanced in table [ServiceLB]. + +Flow 6 is the table-miss flow. Note that packets of NodePort Services have not been identified in this table by matching destination IP address. The identification of NodePort Services will be done finally in table [ServiceLB] by matching `NodePortRegMark` and the @@ -977,8 +1026,10 @@ This table is used to implement Service Endpoint selection. It addresses specifi 3. LoadBalancer, as demonstrated in the example [LoadBalancer]. 4. Service configured with external IPs, as demonstrated in the example [Service with ExternalIP]. 5. Service configured with session affinity, as demonstrated in the example [Service with session affinity]. -6. Service configured with externalTrafficPolicy to `Local`, as demonstrated in the example [Service with +6. Service configured with `externalTrafficPolicy` to `Local`, as demonstrated in the example [Service with ExternalTrafficPolicy Local]. +7. LoadBalancer configured with `loadBalancerSourceRanges`, as demonstrated in the example [LoadBalancer with + LoadBalancerSourceRanges]. If you dump the flows of this table, you may see the following: @@ -993,8 +1044,10 @@ If you dump the flows of this table, you may see the following: eth_type=0x800,nw_proto=6,NXM_OF_TCP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:NXM_NX_REG4[26]->NXM_NX_REG4[26],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9]),\ set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT 8. table=ServiceLB, priority=210,tcp,reg4=0x10010000/0x10070000,nw_dst=192.168.77.151,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x11->reg7,group:17 -9. table=ServiceLB, priority=200,tcp,nw_dst=192.168.77.151,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x12->reg7,group:18 -10. table=ServiceLB, priority=0 actions=goto_table:EndpointDNAT +9. table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=192.168.77.151,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x12->reg7,group:18 +10. table=ServiceLB, priority=210,tcp,reg4=0x10010000/0x10070000,nw_dst=192.168.77.152,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x13->reg7,group:19 +11. table=ServiceLB, priority=200,tcp,reg4=0x20010000/0x20070000,nw_dst=192.168.77.152,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x14->reg7,group:20 +12. table=ServiceLB, priority=0 actions=goto_table:EndpointDNAT ``` Flow 1 and flow 2 are designed for case 1, matching the first packet of connections destined for the sample [ClusterIP @@ -1006,7 +1059,7 @@ loaded, indicating that the source and destination MAC addresses of the packets Service Endpoint selection is not completed yet, as it will be done in the target OVS group. Flow 3 is for case 2, matching the first packet of connections destined for the sample [NodePort]. This is achieved by -matching `EpToSelectRegMark` loaded in table [SessionAffinity], `NodePortRegMark` loaded in table [NodePortMark], and +matching `EpToSelectRegMark` loaded in table [SessionAffinity], `NodePortRegMark` loaded in table [ServiceMark], and NodePort port. Similar to flows 1-2, `RewriteMACRegMark` and `EpSelectedRegMark` are also loaded. Flow 4 is for case 3, processing the first packet of connections destined for the ingress IP of the sample @@ -1034,7 +1087,13 @@ Nodes, even though `externalTrafficPolicy` is set to `Local` for the Service. Du flow 9 exclusively matches packets sourced from the external network, resembling the pattern of flow 1. The target of flow 9 is an OVS group that has only the local Endpoints since `externalTrafficPolicy` of the Service is `Local`. -Flow 10 is the table-miss flow. +Flow 10 and flow 11 are for case 7. Flow 10 is similar to flow 8, while flow 11 diverges slightly from flow 9. The +additional reg mark `LoadBalancerSourceRangesRegMark`, loaded in table [ServiceMark], is used to match the first +packet destined for the sample [LoadBalancer with LoadBalancerSourceRanges], originating from a CIDR listed in the +Service `loadBalancerSourceRanges`. It's worth noting that `loadBalancerSourceRanges` only exclusively influences the +traffic from the external network as `LoadBalancerSourceRangesRegMark` is not used in flow 10. + +Flow 12 is the table-miss flow. As mentioned above, the Service Endpoint selection is performed within OVS groups. 3 typical OVS groups are listed below: @@ -1050,7 +1109,7 @@ As mentioned above, the Service Endpoint selection is performed within OVS group ``` The first group with `group_id` 9 is the destination of packets matched by flow 1, designed for a Service without -Endpoints. The group only has a single bucket where `SvcNoEpRegMark` which will be used in table [EndpointDNAT] is +Endpoints. The group only has a single bucket where `SvcRejectRegMark` which will be used in table [EndpointDNAT] is loaded, indicating that the Service has no Endpoint, and then packets are forwarded to table [EndpointDNAT]. The second group with `group_id` 10 is the destination of packets matched by flow 2, designed for a Service with @@ -1080,7 +1139,7 @@ If you dump the flows of this table, you may see the following:: ``` Flow 1 is designed for Services without Endpoints. It identifies the first packet of connections destined for such Service -by matching `SvcNoEpRegMark`. Subsequently, the packet is forwarded to the OpenFlow controller (Antrea Agent). For TCP +by matching `SvcRejectRegMark`. Subsequently, the packet is forwarded to the OpenFlow controller (Antrea Agent). For TCP Service traffic, the controller will send a TCP RST, and for all other cases the controller will send an ICMP Destination Unreachable message. @@ -1311,7 +1370,7 @@ the following cases when AntreaProxy is not enabled: to complete the DNAT processes, e.g., kube-proxy. The destination MAC of the packets is rewritten in the table to avoid it is forwarded to the original client Pod by mistake. - When hairpin is involved, i.e. connections between 2 local Pods, for which NAT is performed. One example is a - Pod accessing a NodePort Service for which externalTrafficPolicy is set to `Local` using the local Node's IP address, + Pod accessing a NodePort Service for which `externalTrafficPolicy` is set to `Local` using the local Node's IP address, as there will be no SNAT for such traffic. Another example could be hostPort support, depending on how the feature is implemented. @@ -1880,8 +1939,9 @@ Flow 8 is the table-miss flow for case 7. It drops packets that do not match any [L3DecTTL]: #l3decttl [L3Forwarding]: #l3forwarding [LoadBalancer]: #loadbalancer +[LoadBalancer with LoadBalancerSourceRanges]: #loadbalancer-with-loadbalancersourceranges [NodePort]: #nodeport -[NodePortMark]: #nodeportmark +[ServiceMark]: #servicemark [OVS Registers]: #ovs-registers [Output]: #output [PreRoutingClassifier]: #preroutingclassifier diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index e6d3ec5ef87..20b48a438bf 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -799,6 +799,9 @@ func (c *client) InstallServiceFlows(config *types.ServiceConfig) error { if config.IsDSR { flows = append(flows, c.featureService.dsrServiceMarkFlow(config)) } + if len(config.LoadBalancerSourceRanges) != 0 { + flows = append(flows, c.featureService.loadBalancerSourceRangesMarkFlows(config)...) + } cacheKey := generateServicePortFlowCacheKey(config.ServiceIP, config.ServicePort, config.Protocol) return c.addFlows(c.featureService.cachedFlows, cacheKey, flows) } diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 658193b1220..18af9ac3fc3 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -1017,8 +1017,8 @@ func Test_client_GetPodFlowKeys(t *testing.T) { "table=1,priority=200,arp,in_port=11,arp_spa=10.10.0.11,arp_sha=00:00:10:10:00:11", "table=3,priority=190,in_port=11", "table=4,priority=200,ip,in_port=11,dl_src=00:00:10:10:00:11,nw_src=10.10.0.11", - "table=17,priority=200,ip,reg0=0x200/0x200,nw_dst=10.10.0.11", - "table=22,priority=200,dl_dst=00:00:10:10:00:11", + "table=18,priority=200,ip,reg0=0x200/0x200,nw_dst=10.10.0.11", + "table=23,priority=200,dl_dst=00:00:10:10:00:11", } assert.ElementsMatch(t, expectedFlowKeys, flowKeys) } @@ -1254,17 +1254,18 @@ func Test_client_InstallServiceFlows(t *testing.T) { port := uint16(80) testCases := []struct { - name string - trafficPolicyLocal bool - protocol binding.Protocol - svcIP net.IP - affinityTimeout uint16 - isExternal bool - isNodePort bool - isNested bool - isDSR bool - enableMulticluster bool - expectedFlows []string + name string + trafficPolicyLocal bool + protocol binding.Protocol + svcIP net.IP + affinityTimeout uint16 + isExternal bool + isNodePort bool + isNested bool + isDSR bool + enableMulticluster bool + loadBalancerSourceRanges []string + expectedFlows []string }{ { name: "Service ClusterIP", @@ -1449,6 +1450,38 @@ func Test_client_InstallServiceFlows(t *testing.T) { "cookie=0x1030000000064, table=DSRServiceMark, priority=200,tcp6,reg4=0xc000000/0xe000000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,idle_timeout=160,fin_idle_timeout=5,priority=210,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x6,OXM_OF_TCP_SRC[],OXM_OF_TCP_DST[],NXM_NX_IPV6_SRC[],NXM_NX_IPV6_DST[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG4[25],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[]),set_field:0x2000000/0x2000000->reg4,goto_table:EndpointDNAT", }, }, + { + name: "Service LoadBalancer,LoadBalancerSourceRanges,SessionAffinity,Short-circuiting", + protocol: binding.ProtocolSCTP, + svcIP: svcIPv4, + affinityTimeout: uint16(100), + isExternal: true, + trafficPolicyLocal: true, + loadBalancerSourceRanges: []string{"192.168.1.0/24", "192.168.2.0/24"}, + expectedFlows: []string{ + "cookie=0x1030000000000, table=ServiceMark, priority=200,sctp,nw_src=192.168.1.0/24,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=200,sctp,nw_src=192.168.2.0/24,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=190,sctp,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x4000/0x4000->reg0", + "cookie=0x1030000000000, table=ServiceLB, priority=210,sctp,reg4=0x10010000/0x10070000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp,reg4=0x20010000/0x20070000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x65->reg7,group:101", + "cookie=0x1030000000065, table=ServiceLB, priority=190,sctp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000065,eth_type=0x800,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:NXM_NX_REG4[26]->NXM_NX_REG4[26],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + }, + }, + { + name: "Service LoadBalancer,LoadBalancerSourceRanges,IPv6,SessionAffinity", + protocol: binding.ProtocolSCTPv6, + svcIP: svcIPv6, + affinityTimeout: uint16(100), + isExternal: true, + loadBalancerSourceRanges: []string{"fec0:192:168:1::/64", "fec0:192:168:2::/64"}, + expectedFlows: []string{ + "cookie=0x1030000000000, table=ServiceMark, priority=200,sctp6,ipv6_src=fec0:192:168:1::/64,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=200,sctp6,ipv6_src=fec0:192:168:2::/64,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x20000000/0x20000000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=190,sctp6,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x4000/0x4000->reg0", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp6,reg4=0x20010000/0x20070000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:NXM_NX_REG4[26]->NXM_NX_REG4[26],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -1471,17 +1504,18 @@ func Test_client_InstallServiceFlows(t *testing.T) { cacheKey := generateServicePortFlowCacheKey(tc.svcIP, port, tc.protocol) assert.NoError(t, fc.InstallServiceFlows(&types.ServiceConfig{ - ServiceIP: tc.svcIP, - ServicePort: port, - Protocol: tc.protocol, - TrafficPolicyLocal: tc.trafficPolicyLocal, - LocalGroupID: localGroupID, - ClusterGroupID: clusterGroupID, - AffinityTimeout: tc.affinityTimeout, - IsExternal: tc.isExternal, - IsNodePort: tc.isNodePort, - IsNested: tc.isNested, - IsDSR: tc.isDSR, + ServiceIP: tc.svcIP, + ServicePort: port, + Protocol: tc.protocol, + TrafficPolicyLocal: tc.trafficPolicyLocal, + LocalGroupID: localGroupID, + ClusterGroupID: clusterGroupID, + AffinityTimeout: tc.affinityTimeout, + IsExternal: tc.isExternal, + IsNodePort: tc.isNodePort, + IsNested: tc.isNested, + IsDSR: tc.isDSR, + LoadBalancerSourceRanges: tc.loadBalancerSourceRanges, })) fCacheI, ok := fc.featureService.cachedFlows.Load(cacheKey) require.True(t, ok) @@ -1527,11 +1561,11 @@ func Test_client_GetServiceFlowKeys(t *testing.T) { assert.NoError(t, fc.InstallEndpointFlows(bindingProtocol, endpoints)) flowKeys := fc.GetServiceFlowKeys(svcIP, svcPort, bindingProtocol, endpoints) expectedFlowKeys := []string{ - "table=11,priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.224,tp_dst=80", - "table=11,priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.224,tp_dst=80", - "table=12,priority=200,tcp,reg3=0xa0a000b,reg4=0x20050/0x7ffff", - "table=12,priority=200,tcp,reg3=0xa0a000c,reg4=0x20050/0x7ffff", - "table=20,priority=190,ct_state=+new+trk,ip,nw_src=10.10.0.12,nw_dst=10.10.0.12", + "table=12,priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.224,tp_dst=80", + "table=12,priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.224,tp_dst=80", + "table=13,priority=200,tcp,reg3=0xa0a000b,reg4=0x20050/0x7ffff", + "table=13,priority=200,tcp,reg3=0xa0a000c,reg4=0x20050/0x7ffff", + "table=21,priority=190,ct_state=+new+trk,ip,nw_src=10.10.0.12,nw_dst=10.10.0.12", } assert.ElementsMatch(t, expectedFlowKeys, flowKeys) } @@ -2787,8 +2821,8 @@ func Test_client_ReplayFlows(t *testing.T) { "cookie=0x1020000000000, table=IngressMetric, priority=200,reg0=0x400/0x400,reg3=0xf actions=drop", ) replayedFlows = append(replayedFlows, - "cookie=0x1020000000000, table=IngressRule, priority=200,conj_id=15 actions=set_field:0xf->reg3,set_field:0x400/0x400->reg0,set_field:0x800/0x1800->reg0,set_field:0x2000000/0xfe000000->reg0,set_field:0x1b/0xff->reg2,group:4", - "cookie=0x1020000000000, table=IngressDefaultRule, priority=200,reg1=0x64 actions=set_field:0x800/0x1800->reg0,set_field:0x2000000/0xfe000000->reg0,set_field:0x400000/0x600000->reg0,set_field:0x1c/0xff->reg2,goto_table:Output", + "cookie=0x1020000000000, table=IngressRule, priority=200,conj_id=15 actions=set_field:0xf->reg3,set_field:0x400/0x400->reg0,set_field:0x800/0x1800->reg0,set_field:0x2000000/0xfe000000->reg0,set_field:0x1c/0xff->reg2,group:4", + "cookie=0x1020000000000, table=IngressDefaultRule, priority=200,reg1=0x64 actions=set_field:0x800/0x1800->reg0,set_field:0x2000000/0xfe000000->reg0,set_field:0x400000/0x600000->reg0,set_field:0x1d/0xff->reg2,goto_table:Output", ) // Feature Pod connectivity replays flows. diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index 78f0845a143..c54985abc70 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -72,8 +72,8 @@ var ( DispositionPassRegMark = binding.NewRegMark(APDispositionField, DispositionPass) // reg0[13]: Mark to indicate the packet is a generated reject response packet-out. GeneratedRejectPacketOutRegMark = binding.NewOneBitRegMark(0, 13) - // reg0[14]: Mark to indicate a Service without any Endpoints (used by Proxy) - SvcNoEpRegMark = binding.NewOneBitRegMark(0, 14) + // reg0[14]: Mark to indicate a Service connection should be rejected. + SvcRejectRegMark = binding.NewOneBitRegMark(0, 14) // reg0[19]: Mark to indicate remote SNAT for Egress. RemoteSNATRegMark = binding.NewOneBitRegMark(0, 19) // reg0[20]: Field to indicate redirect action of layer 7 NetworkPolicy. @@ -149,6 +149,8 @@ var ( FromExternalRegMark = binding.NewOneBitRegMark(4, 27) // reg4[28]: Mark to indicate that whether the traffic's source is a local Pod or the Node. FromLocalRegMark = binding.NewOneBitRegMark(4, 28) + // reg4[29]: Mark to indicate that whether the LoadBalancer Service traffic's source is included in the Service loadBalancerSourceRanges. + LoadBalancerSourceRangesRegMark = binding.NewOneBitRegMark(4, 29) // reg5(NXM_NX_REG5) // Field to cache the Egress conjunction ID hit by TraceFlow packet. diff --git a/pkg/agent/openflow/framework.go b/pkg/agent/openflow/framework.go index d8c353ee2a2..ab630527b45 100644 --- a/pkg/agent/openflow/framework.go +++ b/pkg/agent/openflow/framework.go @@ -254,6 +254,7 @@ func (f *featureService) getRequiredTables() []*Table { tables := []*Table{ UnSNATTable, PreRoutingClassifierTable, + ServiceMarkTable, SessionAffinityTable, ServiceLBTable, EndpointDNATTable, @@ -263,9 +264,6 @@ func (f *featureService) getRequiredTables() []*Table { ConntrackCommitTable, OutputTable, } - if f.proxyAll { - tables = append(tables, NodePortMarkTable) - } if f.enableDSR { tables = append(tables, DSRServiceMarkTable) } diff --git a/pkg/agent/openflow/framework_test.go b/pkg/agent/openflow/framework_test.go index 2c69c85ffa5..f26032df75c 100644 --- a/pkg/agent/openflow/framework_test.go +++ b/pkg/agent/openflow/framework_test.go @@ -407,7 +407,7 @@ func TestBuildPipeline(t *testing.T) { }, }, { - name: "K8s Node, IPv4 only, with proxyAll enabled", + name: "K8s Node, IPv4 only, with AntreaProxy enabled", ipStack: ipv4Only, features: []feature{ newTestFeaturePodConnectivity(ipStackMap[ipv4Only]), @@ -426,7 +426,7 @@ func TestBuildPipeline(t *testing.T) { ConntrackTable, ConntrackStateTable, PreRoutingClassifierTable, - NodePortMarkTable, + ServiceMarkTable, SessionAffinityTable, ServiceLBTable, EndpointDNATTable, diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index b5f75df9327..8e318d32e80 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -139,7 +139,7 @@ var ( // Tables in stagePreRouting: // When proxy is enabled. PreRoutingClassifierTable = newTable("PreRoutingClassifier", stagePreRouting, pipelineIP) - NodePortMarkTable = newTable("NodePortMark", stagePreRouting, pipelineIP) + ServiceMarkTable = newTable("ServiceMark", stagePreRouting, pipelineIP) SessionAffinityTable = newTable("SessionAffinity", stagePreRouting, pipelineIP) ServiceLBTable = newTable("ServiceLB", stagePreRouting, pipelineIP) DSRServiceMarkTable = newTable("DSRServiceMark", stagePreRouting, pipelineIP) @@ -2288,7 +2288,7 @@ func (f *featureService) nodePortMarkFlows() []binding.Flow { continue } flows = append(flows, - NodePortMarkTable.ofTable.BuildFlow(priorityNormal). + ServiceMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchDstIP(nodePortAddresses[i]). @@ -2298,7 +2298,7 @@ func (f *featureService) nodePortMarkFlows() []binding.Flow { // This generates the flow for the virtual NodePort DNAT IP. The flow is used to mark the first packet of NodePort // connection sourced from the Antrea gateway (the connection is performed DNAT with the virtual IP in host netns). flows = append(flows, - NodePortMarkTable.ofTable.BuildFlow(priorityNormal). + ServiceMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchDstIP(f.virtualNodePortDNATIPs[ipProtocol]). @@ -2408,7 +2408,12 @@ func (f *featureService) serviceLBFlows(config *types.ServiceConfig) []binding.F Action().Group(groupID).Done() } flows := []binding.Flow{ - buildFlow(priorityNormal, config.TrafficPolicyGroupID(), nil), + buildFlow(priorityNormal, config.TrafficPolicyGroupID(), func(b binding.FlowBuilder) binding.FlowBuilder { + if len(config.LoadBalancerSourceRanges) != 0 { + b = b.MatchRegMark(LoadBalancerSourceRangesRegMark) + } + return b + }), } if config.IsExternal && config.TrafficPolicyLocal { // For short-circuiting flow, an extra match condition matching packet from a local Pod or the Node is added. @@ -2551,7 +2556,7 @@ func (f *featureService) serviceEndpointGroup(groupID binding.GroupIDType, withS if len(endpoints) == 0 { return group.Bucket().Weight(100). - LoadRegMark(SvcNoEpRegMark). + LoadRegMark(SvcRejectRegMark). ResubmitToTable(EndpointDNATTable.GetID()). Done() } @@ -3014,7 +3019,7 @@ func (f *featureService) preRoutingClassifierFlows() []binding.Flow { targetTables := []uint8{SessionAffinityTable.GetID(), ServiceLBTable.GetID()} if f.proxyAll { - targetTables = append([]uint8{NodePortMarkTable.GetID()}, targetTables...) + targetTables = append([]uint8{ServiceMarkTable.GetID()}, targetTables...) } for _, ipProtocol := range f.ipProtocols { flows = append(flows, @@ -3107,6 +3112,34 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow { return flows } +func (f *featureService) loadBalancerSourceRangesMarkFlows(config *types.ServiceConfig) []binding.Flow { + cookieID := f.cookieAllocator.Request(f.category).Raw() + protocol := config.Protocol + ingressIP := config.ServiceIP + port := config.ServicePort + var flows []binding.Flow + for _, srcRange := range config.LoadBalancerSourceRanges { + _, srcIPNet, _ := net.ParseCIDR(srcRange) + flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(protocol). + MatchSrcIPNet(*srcIPNet). + MatchDstIP(ingressIP). + MatchDstPort(port, nil). + Action().LoadRegMark(LoadBalancerSourceRangesRegMark). + Done(), + ) + } + flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityLow). + Cookie(cookieID). + MatchProtocol(protocol). + MatchDstIP(ingressIP). + MatchDstPort(port, nil). + Action().LoadRegMark(SvcRejectRegMark). + Done()) + return flows +} + func getCachedFlowMessages(cache *flowCategoryCache) []*openflow15.FlowMod { var flows []*openflow15.FlowMod cache.Range(func(key, value interface{}) bool { diff --git a/pkg/agent/openflow/pipeline_test.go b/pkg/agent/openflow/pipeline_test.go index 451abe7ccc1..e7165e851bb 100644 --- a/pkg/agent/openflow/pipeline_test.go +++ b/pkg/agent/openflow/pipeline_test.go @@ -67,7 +67,8 @@ func pipelineDefaultFlows(egressTrafficShapingEnabled, externalNodeEnabled, isEn "cookie=0x1000000000000, table=UnSNAT, priority=0 actions=goto_table:ConntrackZone", "cookie=0x1000000000000, table=ConntrackZone, priority=0 actions=goto_table:ConntrackState", "cookie=0x1000000000000, table=ConntrackState, priority=0 actions=goto_table:PreRoutingClassifier", - "cookie=0x1000000000000, table=PreRoutingClassifier, priority=0 actions=goto_table:SessionAffinity", + "cookie=0x1000000000000, table=PreRoutingClassifier, priority=0 actions=goto_table:ServiceMark", + "cookie=0x1000000000000, table=ServiceMark, priority=0 actions=goto_table:SessionAffinity", "cookie=0x1000000000000, table=SessionAffinity, priority=0 actions=goto_table:ServiceLB", "cookie=0x1000000000000, table=ServiceLB, priority=0 actions=goto_table:EndpointDNAT", "cookie=0x1000000000000, table=EndpointDNAT, priority=0 actions=goto_table:AntreaPolicyEgressRule", @@ -136,7 +137,8 @@ func pipelineDefaultFlows(egressTrafficShapingEnabled, externalNodeEnabled, isEn "cookie=0x1000000000000, table=UnSNAT, priority=0 actions=goto_table:ConntrackZone", "cookie=0x1000000000000, table=ConntrackZone, priority=0 actions=goto_table:ConntrackState", "cookie=0x1000000000000, table=ConntrackState, priority=0 actions=goto_table:PreRoutingClassifier", - "cookie=0x1000000000000, table=PreRoutingClassifier, priority=0 actions=goto_table:SessionAffinity", + "cookie=0x1000000000000, table=PreRoutingClassifier, priority=0 actions=goto_table:ServiceMark", + "cookie=0x1000000000000, table=ServiceMark, priority=0 actions=goto_table:SessionAffinity", "cookie=0x1000000000000, table=SessionAffinity, priority=0 actions=goto_table:ServiceLB", "cookie=0x1000000000000, table=ServiceLB, priority=0 actions=goto_table:EndpointDNAT", "cookie=0x1000000000000, table=EndpointDNAT, priority=0 actions=goto_table:AntreaPolicyEgressRule", diff --git a/pkg/agent/openflow/service.go b/pkg/agent/openflow/service.go index 2d8a4c0453b..0a30301c0cc 100644 --- a/pkg/agent/openflow/service.go +++ b/pkg/agent/openflow/service.go @@ -135,7 +135,7 @@ func newFeatureService( func (f *featureService) serviceNoEndpointFlow() binding.Flow { return EndpointDNATTable.ofTable.BuildFlow(priorityNormal). Cookie(f.cookieAllocator.Request(f.category).Raw()). - MatchRegMark(SvcNoEpRegMark). + MatchRegMark(SvcRejectRegMark). Action().SendToController([]byte{uint8(PacketInCategorySvcReject)}, false). Done() } diff --git a/pkg/agent/openflow/service_test.go b/pkg/agent/openflow/service_test.go index f8eb226db39..df404bc2df4 100644 --- a/pkg/agent/openflow/service_test.go +++ b/pkg/agent/openflow/service_test.go @@ -49,9 +49,9 @@ func serviceInitFlows(proxyEnabled, isIPv4, proxyAllEnabled, dsrEnabled bool) [] } if proxyAllEnabled { flows = append(flows, - "cookie=0x1030000000000, table=PreRoutingClassifier, priority=200,ip actions=resubmit:NodePortMark,resubmit:SessionAffinity,resubmit:ServiceLB", - "cookie=0x1030000000000, table=NodePortMark, priority=200,ip,nw_dst=192.168.77.100 actions=set_field:0x80000/0x80000->reg4", - "cookie=0x1030000000000, table=NodePortMark, priority=200,ip,nw_dst=169.254.0.252 actions=set_field:0x80000/0x80000->reg4", + "cookie=0x1030000000000, table=PreRoutingClassifier, priority=200,ip actions=resubmit:ServiceMark,resubmit:SessionAffinity,resubmit:ServiceLB", + "cookie=0x1030000000000, table=ServiceMark, priority=200,ip,nw_dst=192.168.77.100 actions=set_field:0x80000/0x80000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=200,ip,nw_dst=169.254.0.252 actions=set_field:0x80000/0x80000->reg4", ) } else { flows = append(flows, @@ -82,9 +82,9 @@ func serviceInitFlows(proxyEnabled, isIPv4, proxyAllEnabled, dsrEnabled bool) [] } if proxyAllEnabled { flows = append(flows, - "cookie=0x1030000000000, table=PreRoutingClassifier, priority=200,ipv6 actions=resubmit:NodePortMark,resubmit:SessionAffinity,resubmit:ServiceLB", - "cookie=0x1030000000000, table=NodePortMark, priority=200,ipv6,ipv6_dst=fec0:192:168:77::100 actions=set_field:0x80000/0x80000->reg4", - "cookie=0x1030000000000, table=NodePortMark, priority=200,ipv6,ipv6_dst=fc01::aabb:ccdd:eefe actions=set_field:0x80000/0x80000->reg4", + "cookie=0x1030000000000, table=PreRoutingClassifier, priority=200,ipv6 actions=resubmit:ServiceMark,resubmit:SessionAffinity,resubmit:ServiceLB", + "cookie=0x1030000000000, table=ServiceMark, priority=200,ipv6,ipv6_dst=fec0:192:168:77::100 actions=set_field:0x80000/0x80000->reg4", + "cookie=0x1030000000000, table=ServiceMark, priority=200,ipv6,ipv6_dst=fc01::aabb:ccdd:eefe actions=set_field:0x80000/0x80000->reg4", ) } else { flows = append(flows, diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 54fecc1db33..392cb0e49c4 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -491,7 +491,8 @@ func serviceIdentityChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { func serviceExternalAddressesChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { return svcInfo.NodePort() != pSvcInfo.NodePort() || !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) || - !slices.Equal(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) + !slices.Equal(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) || + !slices.Equal(svcInfo.LoadBalancerSourceRanges(), pSvcInfo.LoadBalancerSourceRanges()) } // smallSliceDifference builds a slice which includes all the strings from s1 @@ -626,6 +627,7 @@ func (p *proxier) uninstallExternalIPService(svcInfoStr string, externalIPString func (p *proxier) installLoadBalancerService(svcInfoStr string, localGroupID, clusterGroupID binding.GroupIDType, + loadBalancerSourceRanges []string, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, @@ -636,17 +638,18 @@ func (p *proxier) installLoadBalancerService(svcInfoStr string, if ingress != "" { ip := net.ParseIP(ingress) if err := p.ofClient.InstallServiceFlows(&agenttypes.ServiceConfig{ - ServiceIP: ip, - ServicePort: svcPort, - Protocol: protocol, - TrafficPolicyLocal: trafficPolicyLocal, - LocalGroupID: localGroupID, - ClusterGroupID: clusterGroupID, - AffinityTimeout: affinityTimeout, - IsExternal: true, - IsNodePort: false, - IsNested: false, // Unsupported for LoadBalancerIP - IsDSR: features.DefaultFeatureGate.Enabled(features.LoadBalancerModeDSR) && loadBalancerMode == agentconfig.LoadBalancerModeDSR, + ServiceIP: ip, + ServicePort: svcPort, + Protocol: protocol, + TrafficPolicyLocal: trafficPolicyLocal, + LocalGroupID: localGroupID, + ClusterGroupID: clusterGroupID, + AffinityTimeout: affinityTimeout, + IsExternal: true, + IsNodePort: false, + IsNested: false, // Unsupported for LoadBalancerIP + IsDSR: features.DefaultFeatureGate.Enabled(features.LoadBalancerModeDSR) && loadBalancerMode == agentconfig.LoadBalancerModeDSR, + LoadBalancerSourceRanges: loadBalancerSourceRanges, }); err != nil { return fmt.Errorf("failed to install LoadBalancer load balancing flows: %w", err) } @@ -905,7 +908,7 @@ func (p *proxier) installServiceFlows(svcInfo *types.ServiceInfo, localGroupID, } // Install LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { - if err := p.installLoadBalancerService(svcInfoStr, localGroupID, clusterGroupID, svcInfo.LoadBalancerIPStrings(), svcPort, svcProto, svcInfo.ExternalPolicyLocal(), affinityTimeout, loadBalancerMode); err != nil { + if err := p.installLoadBalancerService(svcInfoStr, localGroupID, clusterGroupID, svcInfo.LoadBalancerSourceRanges(), svcInfo.LoadBalancerIPStrings(), svcPort, svcProto, svcInfo.ExternalPolicyLocal(), affinityTimeout, loadBalancerMode); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } @@ -947,13 +950,19 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic } } if p.proxyLoadBalancerIPs { - deletedLoadBalancerIPs := smallSliceDifference(pSvcInfo.LoadBalancerIPStrings(), svcInfo.LoadBalancerIPStrings()) - addedLoadBalancerIPs := smallSliceDifference(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) + var deletedLoadBalancerIPs, addedLoadBalancerIPs []string + if !slices.Equal(svcInfo.LoadBalancerSourceRanges(), pSvcInfo.LoadBalancerSourceRanges()) { + deletedLoadBalancerIPs = pSvcInfo.LoadBalancerIPStrings() + addedLoadBalancerIPs = svcInfo.LoadBalancerIPStrings() + } else { + deletedLoadBalancerIPs = smallSliceDifference(pSvcInfo.LoadBalancerIPStrings(), svcInfo.LoadBalancerIPStrings()) + addedLoadBalancerIPs = smallSliceDifference(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) + } if err := p.uninstallLoadBalancerService(pSvcInfoStr, deletedLoadBalancerIPs, pSvcPort, pSvcProto); err != nil { klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServiceInfo", pSvcInfoStr) return false } - if err := p.installLoadBalancerService(svcInfoStr, localGroupID, clusterGroupID, addedLoadBalancerIPs, svcPort, svcProto, svcInfo.ExternalPolicyLocal(), affinityTimeout, loadBalancerMode); err != nil { + if err := p.installLoadBalancerService(svcInfoStr, localGroupID, clusterGroupID, svcInfo.LoadBalancerSourceRanges(), addedLoadBalancerIPs, svcPort, svcProto, svcInfo.ExternalPolicyLocal(), affinityTimeout, loadBalancerMode); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 485a9695932..608f5f5b560 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -54,23 +54,25 @@ import ( ) var ( - svc1IPv4 = net.ParseIP("10.20.30.41") - svc2IPv4 = net.ParseIP("10.20.30.42") - svc1IPv6 = net.ParseIP("2001::10:20:30:41") - ep1IPv4 = net.ParseIP("10.180.0.1") - ep1IPv6 = net.ParseIP("2001::10:180:0:1") - ep2IPv4 = net.ParseIP("10.180.0.2") - ep2IPv6 = net.ParseIP("2001::10:180:0:2") - loadBalancerIPv4 = net.ParseIP("169.254.169.1") - loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") - loadBalancerIPModeProxyIPv4 = net.ParseIP("169.254.169.2") - loadBalancerIPModeProxyIPv6 = net.ParseIP("fec0::169:254:169:2") - svcNodePortIPv4 = net.ParseIP("192.168.77.100") - svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") - externalIPv4 = net.ParseIP("192.168.77.101") - externalIPv6 = net.ParseIP("2001::192:168:77:101") - nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} - nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} + svc1IPv4 = net.ParseIP("10.20.30.41") + svc2IPv4 = net.ParseIP("10.20.30.42") + svc1IPv6 = net.ParseIP("2001::10:20:30:41") + ep1IPv4 = net.ParseIP("10.180.0.1") + ep1IPv6 = net.ParseIP("2001::10:180:0:1") + ep2IPv4 = net.ParseIP("10.180.0.2") + ep2IPv6 = net.ParseIP("2001::10:180:0:2") + loadBalancerIPv4 = net.ParseIP("169.254.169.1") + loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") + loadBalancerIPModeProxyIPv4 = net.ParseIP("169.254.169.2") + loadBalancerIPModeProxyIPv6 = net.ParseIP("fec0::169:254:169:2") + loadBalancerSourceRangesIPv4 = []string{"192.168.1.0/24", "192.168.2.0/24"} + loadBalancerSourceRangesIPv6 = []string{"fec0:192:168:1::/64", "fec0:192:168:2::/64"} + svcNodePortIPv4 = net.ParseIP("192.168.77.100") + svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") + externalIPv4 = net.ParseIP("192.168.77.101") + externalIPv6 = net.ParseIP("2001::192:168:77:101") + nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} + nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} svcPort = 80 svcNodePort = 30008 @@ -227,6 +229,7 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + loadBalancerSourceRanges []string, externalIPs, loadBalancerIPs []net.IP, loadBalancerIPModeProxyIPs []net.IP, @@ -239,6 +242,7 @@ func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, return makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { svc.Spec.ClusterIP = clusterIP.String() svc.Spec.Type = corev1.ServiceTypeLoadBalancer + svc.Spec.LoadBalancerSourceRanges = loadBalancerSourceRanges var ingress []corev1.LoadBalancerIngress for _, ip := range loadBalancerIPs { if ip != nil { @@ -578,6 +582,7 @@ func testLoadBalancerAdd(t *testing.T, ep2IP := ep2IPv4 loadBalancerIP := loadBalancerIPv4 loadBalancerIPModeProxyIP := loadBalancerIPModeProxyIPv4 + loadBalancerSourceRanges := loadBalancerSourceRangesIPv4 if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 vIP = agentconfig.VirtualNodePortDNATIPv6 @@ -588,6 +593,7 @@ func testLoadBalancerAdd(t *testing.T, ep2IP = ep2IPv6 loadBalancerIP = loadBalancerIPv6 loadBalancerIPModeProxyIP = loadBalancerIPModeProxyIPv6 + loadBalancerSourceRanges = loadBalancerSourceRangesIPv6 } fp := newFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) @@ -601,6 +607,7 @@ func testLoadBalancerAdd(t *testing.T, } svc := makeTestLoadBalancerService(&svcPortName, svcIP, + loadBalancerSourceRanges, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, @@ -662,14 +669,15 @@ func testLoadBalancerAdd(t *testing.T, }).Times(1) if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ - ServiceIP: loadBalancerIP, - ServicePort: uint16(svcPort), - Protocol: bindingProtocol, - TrafficPolicyLocal: nodeLocalExternal, - LocalGroupID: 1, - ClusterGroupID: 2, - IsExternal: true, - IsDSR: isDSR, + ServiceIP: loadBalancerIP, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + TrafficPolicyLocal: nodeLocalExternal, + LocalGroupID: 1, + ClusterGroupID: 2, + IsExternal: true, + IsDSR: isDSR, + LoadBalancerSourceRanges: loadBalancerSourceRanges, }).Times(1) } if externalIP != nil { @@ -721,14 +729,15 @@ func testLoadBalancerAdd(t *testing.T, }).Times(1) if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ - ServiceIP: loadBalancerIP, - ServicePort: uint16(svcPort), - Protocol: bindingProtocol, - TrafficPolicyLocal: nodeLocalExternal, - LocalGroupID: localGroupID, - ClusterGroupID: clusterGroupID, - IsExternal: true, - IsDSR: isDSR, + ServiceIP: loadBalancerIP, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + TrafficPolicyLocal: nodeLocalExternal, + LocalGroupID: localGroupID, + ClusterGroupID: clusterGroupID, + IsExternal: true, + IsDSR: isDSR, + LoadBalancerSourceRanges: loadBalancerSourceRanges, }).Times(1) } if externalIP != nil { @@ -1666,6 +1675,7 @@ func testLoadBalancerRemove(t *testing.T, bindingProtocol binding.Protocol, isIP epIP := ep1IPv4 loadBalancerIP := loadBalancerIPv4 loadBalancerIPModeProxyIP := loadBalancerIPModeProxyIPv4 + loadBalancerSourceRanges := loadBalancerSourceRangesIPv4 if isIPv6 { vIP = agentconfig.VirtualNodePortDNATIPv6 svcNodePortIP = svcNodePortIPv6 @@ -1675,6 +1685,7 @@ func testLoadBalancerRemove(t *testing.T, bindingProtocol binding.Protocol, isIP epIP = ep1IPv6 loadBalancerIP = loadBalancerIPv6 loadBalancerIPModeProxyIP = loadBalancerIPModeProxyIPv6 + loadBalancerSourceRanges = loadBalancerSourceRangesIPv6 } fp := newFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) @@ -1683,6 +1694,7 @@ func testLoadBalancerRemove(t *testing.T, bindingProtocol binding.Protocol, isIP svc := makeTestLoadBalancerService(&svcPortName, svcIP, + loadBalancerSourceRanges, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, @@ -1727,13 +1739,14 @@ func testLoadBalancerRemove(t *testing.T, bindingProtocol binding.Protocol, isIP IsNodePort: true, }).Times(1) mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ - ServiceIP: loadBalancerIP, - ServicePort: uint16(svcPort), - Protocol: bindingProtocol, - TrafficPolicyLocal: true, - LocalGroupID: 1, - ClusterGroupID: 2, - IsExternal: true, + ServiceIP: loadBalancerIP, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + TrafficPolicyLocal: true, + LocalGroupID: 1, + ClusterGroupID: 2, + IsExternal: true, + LoadBalancerSourceRanges: loadBalancerSourceRanges, }).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) @@ -2080,6 +2093,7 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, + nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), @@ -2091,6 +2105,7 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, + nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort+1), @@ -2269,6 +2284,7 @@ func testLoadBalancerRemoveEndpoints(t *testing.T, nodePortAddresses []net.IP, s svc := makeTestLoadBalancerService(&svcPortName, svcIP, + nil, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, @@ -2504,8 +2520,8 @@ func testServicePortUpdate(t *testing.T, svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2639,8 +2655,8 @@ func testServiceNodePortUpdate(t *testing.T, svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2755,8 +2771,8 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, svc = makeTestNodePortService(&svcPortName, svcIP, []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) updatedSvc = makeTestNodePortService(&svcPortName, svcIP, []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{externalIP}, []net.IP{loadBalancerIP}, []net.IP{loadBalancerIPModeProxyIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) } makeServiceMap(fp, svc) @@ -3002,6 +3018,119 @@ func TestServiceInternalTrafficPolicyUpdate(t *testing.T) { }) } +func testServiceLoadBalancerSourceRangesUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + epIP net.IP, + loadBalancerIPs []net.IP, + updatedLoadBalancerIPs []net.IP, + loadBalancerSourceRanges []string, + updatedLoadBalancerSourceRanges []string, + isIPv6 bool) { + ctrl := gomock.NewController(t) + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator() + fp := newFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + svc := makeTestLoadBalancerService(&svcPortName, svcIP, loadBalancerSourceRanges, nil, loadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, updatedLoadBalancerSourceRanges, nil, updatedLoadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + makeServiceMap(fp, svc) + + ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, []discovery.Endpoint{*ep}, []discovery.EndpointPort{*epPort}, isIPv6) + makeEndpointSliceMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, true, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(binding.GroupIDType(1), false, gomock.InAnyOrder(expectedEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ + ServiceIP: svcIP, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + ClusterGroupID: 1, + }).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ + ServiceIP: vIP, + ServicePort: uint16(svcNodePort), + Protocol: bindingProtocol, + ClusterGroupID: 1, + IsExternal: true, + IsNodePort: true, + }).Times(1) + for _, ip := range loadBalancerIPs { + mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ + ServiceIP: ip, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + ClusterGroupID: 1, + IsExternal: true, + LoadBalancerSourceRanges: loadBalancerSourceRanges, + }).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(ip).Times(1) + } + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + + for _, ip := range loadBalancerIPs { + mockOFClient.EXPECT().UninstallServiceFlows(ip, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(ip).Times(1) + } + + for _, ip := range updatedLoadBalancerIPs { + mockOFClient.EXPECT().InstallServiceFlows(&antreatypes.ServiceConfig{ + ServiceIP: ip, + ServicePort: uint16(svcPort), + Protocol: bindingProtocol, + ClusterGroupID: 1, + IsExternal: true, + LoadBalancerSourceRanges: updatedLoadBalancerSourceRanges, + }).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(ip).Times(1) + } + + fp.syncProxyRules() + assert.Contains(t, fp.serviceInstalledMap, svcPortName) + assert.Contains(t, fp.endpointsInstalledMap, svcPortName) + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() + assert.Contains(t, fp.serviceInstalledMap, svcPortName) + assert.Contains(t, fp.endpointsInstalledMap, svcPortName) +} + +func TestServiceLoadBalancerSourceRangesUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + loadBalancerIPs := []net.IP{net.ParseIP("169.254.1.1"), net.ParseIP("169.254.1.2")} + updatedLoadBalancerIPs := []net.IP{net.ParseIP("169.254.1.2"), net.ParseIP("169.254.1.3")} + loadBalancerSourceRanges := []string{"192.168.1.0/24", "192.168.2.0/24"} + updatedLoadBalancerSourceRanges := []string{"192.168.3.0/24", "192.168.4.0/24"} + t.Run("LoadBalancer ingress IPs update", func(t *testing.T) { + testServiceLoadBalancerSourceRangesUpdate(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPs, updatedLoadBalancerIPs, loadBalancerSourceRanges, updatedLoadBalancerSourceRanges, false) + }) + t.Run("LoadBalancer ingress IPs remain", func(t *testing.T) { + testServiceLoadBalancerSourceRangesUpdate(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPs, loadBalancerIPs, loadBalancerSourceRanges, updatedLoadBalancerSourceRanges, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + loadBalancerIPs := []net.IP{net.ParseIP("fec0::169:254:1:1"), net.ParseIP("fec0::169:254:1:2")} + updatedLoadBalancerIPs := []net.IP{net.ParseIP("fec0::169:254:1:2"), net.ParseIP("fec0::169:254:1:3")} + loadBalancerSourceRanges := []string{"fec0:192:168:1::/64", "fec0:192:168:2::/64"} + updatedLoadBalancerSourceRanges := []string{"fec0:192:168:3::/64", "fec0:192:168:4::/64"} + t.Run("LoadBalancer ingress IPs update", func(t *testing.T) { + testServiceLoadBalancerSourceRangesUpdate(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPs, updatedLoadBalancerIPs, loadBalancerSourceRanges, updatedLoadBalancerSourceRanges, true) + }) + t.Run("LoadBalancer ingress IPs remain", func(t *testing.T) { + testServiceLoadBalancerSourceRangesUpdate(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPs, loadBalancerIPs, loadBalancerSourceRanges, updatedLoadBalancerSourceRanges, true) + }) + }) +} + func testServiceIngressIPsUpdate(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, @@ -3022,8 +3151,8 @@ func testServiceIngressIPsUpdate(t *testing.T, updatedLoadBalancerIPStrs = append(updatedLoadBalancerIPStrs, ip.String()) } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, loadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, updatedLoadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, loadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, updatedLoadBalancerIPs, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -3135,8 +3264,8 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -3274,8 +3403,8 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, nil, []net.IP{loadBalancerIP}, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) diff --git a/pkg/agent/types/service.go b/pkg/agent/types/service.go index 3f76631475c..5c1433d80f3 100644 --- a/pkg/agent/types/service.go +++ b/pkg/agent/types/service.go @@ -36,7 +36,8 @@ type ServiceConfig struct { // IsNested indicates the whether Service's Endpoints are ClusterIPs of other Services. It's used in multi-cluster. IsNested bool // IsDSR indicates that whether the Service works in Direct Server Return mode. - IsDSR bool + IsDSR bool + LoadBalancerSourceRanges []string } func (c *ServiceConfig) TrafficPolicyGroupID() openflow.GroupIDType {