Add natoutgoing policy rules (#2883)

* natoutgoine rules

* add e2e

charts/templates/kube-ovn-crd.yaml

@@ -1593,6 +1593,25 @@ spec:
                   type: string
                 v6availableIPrange:
                   type: string
+                natOutgoingPolicyRules:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      ruleID:
+                        type: string
+                      action:
+                        type: string
+                        enum:
+                          - nat
+                          - forward
+                      match:
+                        type: object
+                        properties:
+                          srcIPs:
+                            type: string
+                          dstIPs:
+                            type: string
                 conditions:
                   type: array
                   items:
@@ -1711,6 +1730,23 @@ spec:
                           - allow
                           - drop
                           - reject
+                natOutgoingPolicyRules:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      action:
+                        type: string
+                        enum:
+                          - nat
+                          - forward
+                      match:
+                        type: object
+                        properties:
+                          srcIPs:
+                            type: string
+                          dstIPs:
+                            type: string
                 u2oInterconnection:
                   type: boolean
                 enableLb:

dist/images/install.sh

@@ -2131,6 +2131,25 @@ spec:
                   type: string
                 v6availableIPrange:
                   type: string
+                natOutgoingPolicyRules:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      ruleID:
+                        type: string
+                      action:
+                        type: string
+                        enum:
+                          - nat
+                          - forward
+                      match:
+                        type: object
+                        properties:
+                          srcIPs:
+                            type: string
+                          dstIPs:
+                            type: string
                 conditions:
                   type: array
                   items:
@@ -2249,6 +2268,23 @@ spec:
                           - allow
                           - drop
                           - reject
+                natOutgoingPolicyRules:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      action:
+                        type: string
+                        enum:
+                          - nat
+                          - forward
+                      match:
+                        type: object
+                        properties:
+                          srcIPs:
+                            type: string
+                          dstIPs:
+                            type: string
                 u2oInterconnection:
                   type: boolean
                 enableLb:

dist/images/uninstall.sh

@@ -8,6 +8,8 @@ iptables -t nat -F OVN-PREROUTING
 iptables -t nat -X OVN-PREROUTING
 iptables -t nat -F OVN-POSTROUTING
 iptables -t nat -X OVN-POSTROUTING
+iptables -t nat -F OVN-NAT-POLICY
+iptables -t nat -X OVN-NAT-POLICY
 iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
 iptables -t filter -D INPUT -m set --match-set ovn40subnets src -j ACCEPT
 iptables -t filter -D INPUT -m set --match-set ovn40services dst -j ACCEPT
@@ -26,13 +28,16 @@ ipset destroy ovn40subnets-distributed-gw
 ipset destroy ovn40local-pod-ip-nat
 ipset destroy ovn40other-node
 ipset destroy ovn40services
+ipset destroy ovn40subnets-nat-policy
 
 ip6tables -t nat -D PREROUTING -j OVN-PREROUTING -m comment --comment "kube-ovn prerouting rules"
 ip6tables -t nat -D POSTROUTING -j OVN-POSTROUTING -m comment --comment "kube-ovn postrouting rules"
 ip6tables -t nat -F OVN-PREROUTING
 ip6tables -t nat -X OVN-PREROUTING
 ip6tables -t nat -F OVN-POSTROUTING
 ip6tables -t nat -X OVN-POSTROUTING
+ip6tables -t nat -F OVN-NAT-POLICY
+ip6tables -t nat -X OVN-NAT-POLICY
 ip6tables -t filter -D INPUT -m set --match-set ovn60subnets dst -j ACCEPT
 ip6tables -t filter -D INPUT -m set --match-set ovn60subnets src -j ACCEPT
 ip6tables -t filter -D INPUT -m set --match-set ovn60services dst -j ACCEPT
@@ -51,6 +56,7 @@ ipset destroy ovn60subnets-distributed-gw
 ipset destroy ovn60local-pod-ip-nat
 ipset destroy ovn60other-node
 ipset destroy ovn60services
+ipset destroy ovn60subnets-nat-policy
 
 rm -rf /var/run/openvswitch/*
 rm -rf /var/run/ovn/*

pkg/apis/kubeovn/v1/types.go

@@ -159,6 +159,8 @@ type SubnetSpec struct {
 
 	Acls []Acl `json:"acls,omitempty"`
 
+	NatOutgoingPolicyRules []NatOutgoingPolicyRule `json:"natOutgoingPolicyRules,omitempty"`
+
 	U2OInterconnection bool  `json:"u2oInterconnection,omitempty"`
 	EnableLb           *bool `json:"enableLb,omitempty"`
 	EnableEcmp         bool  `json:"enableEcmp,omitempty"`
@@ -173,6 +175,21 @@ type Acl struct {
 	Action    string `json:"action,omitempty"`
 }
 
+type NatOutgoingPolicyRule struct {
+	Match  NatOutGoingPolicyMatch `json:"match"`
+	Action string                 `json:"action"`
+}
+
+type NatOutgoingPolicyRuleStatus struct {
+	RuleID string `json:"ruleID"`
+	NatOutgoingPolicyRule
+}
+
+type NatOutGoingPolicyMatch struct {
+	SrcIPs string `json:"srcIPs,omitempty"`
+	DstIPs string `json:"dstIPs,omitempty"`
+}
+
 // ConditionType encodes information on the condition
 type ConditionType string
 
@@ -204,19 +221,20 @@ type SubnetStatus struct {
 	// +patchStrategy=merge
 	Conditions []SubnetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
 
-	V4AvailableIPs        float64 `json:"v4availableIPs"`
-	V4AvailableIPRange    string  `json:"v4availableIPrange"`
-	V4UsingIPs            float64 `json:"v4usingIPs"`
-	V4UsingIPRange        string  `json:"v4usingIPrange"`
-	V6AvailableIPs        float64 `json:"v6availableIPs"`
-	V6AvailableIPRange    string  `json:"v6availableIPrange"`
-	V6UsingIPs            float64 `json:"v6usingIPs"`
-	V6UsingIPRange        string  `json:"v6usingIPrange"`
-	ActivateGateway       string  `json:"activateGateway"`
-	DHCPv4OptionsUUID     string  `json:"dhcpV4OptionsUUID"`
-	DHCPv6OptionsUUID     string  `json:"dhcpV6OptionsUUID"`
-	U2OInterconnectionIP  string  `json:"u2oInterconnectionIP"`
-	U2OInterconnectionVPC string  `json:"u2oInterconnectionVPC"`
+	V4AvailableIPs         float64                       `json:"v4availableIPs"`
+	V4AvailableIPRange     string                        `json:"v4availableIPrange"`
+	V4UsingIPs             float64                       `json:"v4usingIPs"`
+	V4UsingIPRange         string                        `json:"v4usingIPrange"`
+	V6AvailableIPs         float64                       `json:"v6availableIPs"`
+	V6AvailableIPRange     string                        `json:"v6availableIPrange"`
+	V6UsingIPs             float64                       `json:"v6usingIPs"`
+	V6UsingIPRange         string                        `json:"v6usingIPrange"`
+	ActivateGateway        string                        `json:"activateGateway"`
+	DHCPv4OptionsUUID      string                        `json:"dhcpV4OptionsUUID"`
+	DHCPv6OptionsUUID      string                        `json:"dhcpV6OptionsUUID"`
+	U2OInterconnectionIP   string                        `json:"u2oInterconnectionIP"`
+	U2OInterconnectionVPC  string                        `json:"u2oInterconnectionVPC"`
+	NatOutgoingPolicyRules []NatOutgoingPolicyRuleStatus `json:"natOutgoingPolicyRules"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/controller/subnet.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"reflect"
@@ -101,7 +102,8 @@ func (c *Controller) enqueueUpdateSubnet(old, new interface{}) {
 		!reflect.DeepEqual(oldSubnet.Spec.Acls, newSubnet.Spec.Acls) ||
 		oldSubnet.Spec.U2OInterconnection != newSubnet.Spec.U2OInterconnection ||
 		oldSubnet.Spec.RouteTable != newSubnet.Spec.RouteTable ||
-		oldSubnet.Spec.Vpc != newSubnet.Spec.Vpc {
+		oldSubnet.Spec.Vpc != newSubnet.Spec.Vpc ||
+		!reflect.DeepEqual(oldSubnet.Spec.NatOutgoingPolicyRules, newSubnet.Spec.NatOutgoingPolicyRules) {
 		klog.V(3).Infof("enqueue update subnet %s", key)
 
 		if oldSubnet.Spec.GatewayType != newSubnet.Spec.GatewayType {
@@ -320,6 +322,34 @@ func formatSubnet(subnet *kubeovnv1.Subnet, c *Controller) error {
 	return nil
 }
 
+func genNatOutgoingPolicyRulesStatus(subnet *kubeovnv1.Subnet) error {
+	subnet.Status.NatOutgoingPolicyRules = make([]kubeovnv1.NatOutgoingPolicyRuleStatus, len(subnet.Spec.NatOutgoingPolicyRules))
+
+	if len(subnet.Spec.NatOutgoingPolicyRules) != 0 {
+		for index, rule := range subnet.Spec.NatOutgoingPolicyRules {
+			jsonRule, err := json.Marshal(rule)
+			if err != nil {
+				return err
+			}
+			priority := fmt.Sprintf("%d", index)
+			// hash code generate by subnetName, rule and priority
+			var retBytes []byte
+			retBytes = append(retBytes, []byte(subnet.Name)...)
+			retBytes = append(retBytes, []byte(priority)...)
+			retBytes = append(retBytes, jsonRule...)
+			result := util.Sha256ByteToString(retBytes)
+
+			subnet.Status.NatOutgoingPolicyRules[index].RuleID = result[:util.NatPolicyRuleIDLength]
+			subnet.Status.NatOutgoingPolicyRules[index].Match = rule.Match
+			subnet.Status.NatOutgoingPolicyRules[index].Action = rule.Action
+		}
+	} else {
+		subnet.Status.NatOutgoingPolicyRules = nil
+	}
+
+	return nil
+}
+
 func checkSubnetChanged(subnet *kubeovnv1.Subnet) (bool, error) {
 	var err error
 	changed := false
@@ -643,6 +673,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
 		c.patchSubnetStatus(subnet, "ValidateLogicalSwitchSuccess", "")
 	}
 
+	if err := genNatOutgoingPolicyRulesStatus(subnet); err != nil {
+		klog.Error(err)
+		return err
+	}
+
 	if subnet.Spec.Protocol == kubeovnv1.ProtocolDual {
 		err = calcDualSubnetStatusIP(subnet, c)
 	} else {

pkg/daemon/controller_linux.go

@@ -22,6 +22,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
+	k8sipset "k8s.io/kubernetes/pkg/util/ipset"
 	k8siptables "k8s.io/kubernetes/pkg/util/iptables"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
@@ -34,6 +35,7 @@ type ControllerRuntime struct {
 	iptables         map[string]*iptables.IPTables
 	iptablesObsolete map[string]*iptables.IPTables
 	k8siptables      map[string]k8siptables.Interface
+	k8sipsets        k8sipset.Interface
 	ipsets           map[string]*ipsets.IPSets
 	gwCounters       map[string]*util.GwIPtableCounters
 }
@@ -78,6 +80,7 @@ func (c *Controller) initRuntime() error {
 	c.ipsets = make(map[string]*ipsets.IPSets)
 	c.gwCounters = make(map[string]*util.GwIPtableCounters)
 	c.k8siptables = make(map[string]k8siptables.Interface)
+	c.k8sipsets = k8sipset.New(c.k8sExec)
 
 	if c.protocol == kubeovnv1.ProtocolIPv4 || c.protocol == kubeovnv1.ProtocolDual {
 		ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)

pkg/daemon/gateway.go

@@ -34,7 +34,7 @@ func (c *Controller) runGateway() {
 	if err := c.setExGateway(); err != nil {
 		klog.Errorf("failed to set ex gateway, %v", err)
 	}
-
+	c.gcIPSet()
 	c.appendMssRule()
 }
 
@@ -83,6 +83,18 @@ func (c *Controller) setICGateway() error {
 	return nil
 }
 
+func (c *Controller) isSubnetNeedNat(subnet *kubeovnv1.Subnet, protocol string) bool {
+	if subnet.DeletionTimestamp == nil &&
+		subnet.Spec.NatOutgoing &&
+		(subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) &&
+		subnet.Spec.Vpc == c.config.ClusterRouter &&
+		subnet.Spec.CIDRBlock != "" &&
+		(subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {
+		return true
+	}
+	return false
+}
+
 func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) {
 	var subnetsNeedNat []string
 	subnets, err := c.subnetsLister.List(labels.Everything())
@@ -92,19 +104,30 @@ func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) {
 	}
 
 	for _, subnet := range subnets {
-		if subnet.DeletionTimestamp == nil &&
-			subnet.Spec.NatOutgoing &&
-			(subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) &&
-			subnet.Spec.Vpc == c.config.ClusterRouter &&
-			subnet.Spec.CIDRBlock != "" &&
-			(subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {
+		if c.isSubnetNeedNat(subnet, protocol) {
 			cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)
 			subnetsNeedNat = append(subnetsNeedNat, cidrBlock)
 		}
 	}
 	return subnetsNeedNat, nil
 }
 
+func (c *Controller) getSubnetsNatOutGoingPolicy(protocol string) ([]*kubeovnv1.Subnet, error) {
+	subnets, err := c.subnetsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("list subnets failed, %v", err)
+		return nil, err
+	}
+
+	var subnetsWithNatPolicy []*kubeovnv1.Subnet
+	for _, subnet := range subnets {
+		if c.isSubnetNeedNat(subnet, protocol) && len(subnet.Spec.NatOutgoingPolicyRules) != 0 {
+			subnetsWithNatPolicy = append(subnetsWithNatPolicy, subnet)
+		}
+	}
+	return subnetsWithNatPolicy, nil
+}
+
 func (c *Controller) getSubnetsDistributedGateway(protocol string) ([]string, error) {
 	subnets, err := c.subnetsLister.List(labels.Everything())
 	if err != nil {