From f747a4babbc408665b1791ef3fbb75c56044e87f Mon Sep 17 00:00:00 2001 From: Shufang Date: Tue, 16 Jun 2020 00:17:04 -0700 Subject: [PATCH 1/3] Move Cidr translation from iptable to ipset. --- npm/ipsm/ipsm.go | 29 ++---- npm/ipsm/ipsm_test.go | 17 ++-- npm/namespace.go | 2 +- npm/npm.go | 2 +- npm/nwpolicy.go | 61 ++++++++++-- npm/nwpolicy_test.go | 6 +- npm/pod.go | 2 +- npm/translatePolicy.go | 182 ++++++++++++++++++------------------ npm/translatePolicy_test.go | 118 +++++++++++------------ npm/util/const.go | 5 + 10 files changed, 232 insertions(+), 192 deletions(-) diff --git a/npm/ipsm/ipsm.go b/npm/ipsm/ipsm.go index 0eaf8c5042..6f4c9df183 100644 --- a/npm/ipsm/ipsm.go +++ b/npm/ipsm/ipsm.go @@ -16,7 +16,7 @@ type ipsEntry struct { operationFlag string name string set string - spec string + spec []string } // IpsetManager stores ipset states. @@ -81,7 +81,7 @@ func (ipsMgr *IpsetManager) CreateList(listName string) error { name: listName, operationFlag: util.IpsetCreationFlag, set: util.GetHashedName(listName), - spec: util.IpsetSetListFlag, + spec: append([]string{util.IpsetSetListFlag}), } log.Printf("Creating List: %+v", entry) if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { @@ -132,7 +132,7 @@ func (ipsMgr *IpsetManager) AddToList(listName string, setName string) error { entry := &ipsEntry{ operationFlag: util.IpsetAppendFlag, set: util.GetHashedName(listName), - spec: util.GetHashedName(setName), + spec: append([]string{util.GetHashedName(setName)}), } if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { @@ -162,7 +162,7 @@ func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) erro entry := &ipsEntry{ operationFlag: util.IpsetDeletionFlag, set: hashedListName, - spec: hashedSetName, + spec: append([]string{hashedSetName}), } if _, err := ipsMgr.Run(entry); err != nil { @@ -181,7 +181,7 @@ func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) erro } // CreateSet creates an ipset. -func (ipsMgr *IpsetManager) CreateSet(setName, spec string) error { +func (ipsMgr *IpsetManager) CreateSet(setName string, spec []string) error { if _, exists := ipsMgr.setMap[setName]; exists { return nil } @@ -211,10 +211,6 @@ func (ipsMgr *IpsetManager) DeleteSet(setName string) error { return nil } - if len(ipsMgr.setMap[setName].elements) > 0 { - return nil - } - entry := &ipsEntry{ operationFlag: util.IpsetDestroyFlag, set: util.GetHashedName(setName), @@ -240,14 +236,14 @@ func (ipsMgr *IpsetManager) AddToSet(setName, ip, spec string) error { return nil } - if err := ipsMgr.CreateSet(setName, spec); err != nil { + if err := ipsMgr.CreateSet(setName, append([]string{util.IpsetNetHashFlag})); err != nil { return err } entry := &ipsEntry{ operationFlag: util.IpsetAppendFlag, set: util.GetHashedName(setName), - spec: ip, + spec: append([]string{ip}), } if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { @@ -276,7 +272,7 @@ func (ipsMgr *IpsetManager) DeleteFromSet(setName, ip string) error { entry := &ipsEntry{ operationFlag: util.IpsetDeletionFlag, set: util.GetHashedName(setName), - spec: ip, + spec: append([]string{ip}), } if errCode, err := ipsMgr.Run(entry); err != nil { @@ -342,13 +338,8 @@ func (ipsMgr *IpsetManager) Destroy() error { // Run execute an ipset command to update ipset. func (ipsMgr *IpsetManager) Run(entry *ipsEntry) (int, error) { cmdName := util.Ipset - cmdArgs := []string{entry.operationFlag, util.IpsetExistFlag} - if len(entry.set) > 0 { - cmdArgs = append(cmdArgs, entry.set) - } - if len(entry.spec) > 0 { - cmdArgs = append(cmdArgs, entry.spec) - } + cmdArgs := append([]string{entry.operationFlag, util.IpsetExistFlag, entry.set}, entry.spec...) + cmdArgs = util.DropEmptyFields(cmdArgs) log.Printf("Executing ipset command %s %v", cmdName, cmdArgs) _, err := exec.Command(cmdName, cmdArgs...).Output() diff --git a/npm/ipsm/ipsm_test.go b/npm/ipsm/ipsm_test.go index c90fe37d72..7bf968c886 100644 --- a/npm/ipsm/ipsm_test.go +++ b/npm/ipsm/ipsm_test.go @@ -77,7 +77,7 @@ func TestAddToList(t *testing.T) { } }() - if err := ipsMgr.CreateSet("test-set", util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("test-set", append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestAddToList failed @ ipsMgr.CreateSet") } @@ -98,7 +98,7 @@ func TestDeleteFromList(t *testing.T) { } }() - if err := ipsMgr.CreateSet("test-set", util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("test-set", append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestDeleteFromList failed @ ipsMgr.CreateSet") } @@ -127,9 +127,14 @@ func TestCreateSet(t *testing.T) { } }() - if err := ipsMgr.CreateSet("test-set", util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("test-set", []string{util.IpsetNetHashFlag}); err != nil { t.Errorf("TestCreateSet failed @ ipsMgr.CreateSet") } + + spec := append([]string{util.IpsetNetHashFlag, util.IpsetMaxelemName, util.IpsetMaxelemNum}) + if err := ipsMgr.CreateSet("test-set-with-maxelem", spec); err != nil { + t.Errorf("TestCreateSet failed @ ipsMgr.CreateSet when set maxelem") + } } func TestDeleteSet(t *testing.T) { @@ -144,7 +149,7 @@ func TestDeleteSet(t *testing.T) { } }() - if err := ipsMgr.CreateSet("test-set", util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("test-set", append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestDeleteSet failed @ ipsMgr.CreateSet") } @@ -203,7 +208,7 @@ func TestClean(t *testing.T) { } }() - if err := ipsMgr.CreateSet("test-set", util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("test-set", append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestClean failed @ ipsMgr.CreateSet") } @@ -248,7 +253,7 @@ func TestRun(t *testing.T) { entry := &ipsEntry{ operationFlag: util.IpsetCreationFlag, set: "test-set", - spec: util.IpsetNetHashFlag, + spec: append([]string{util.IpsetNetHashFlag}), } if _, err := ipsMgr.Run(entry); err != nil { t.Errorf("TestRun failed @ ipsMgr.Run") diff --git a/npm/namespace.go b/npm/namespace.go index 95bb18bbf6..5694be98f9 100644 --- a/npm/namespace.go +++ b/npm/namespace.go @@ -106,7 +106,7 @@ func (npMgr *NetworkPolicyManager) AddNamespace(nsObj *corev1.Namespace) error { ipsMgr := npMgr.nsMap[util.KubeAllNamespacesFlag].ipsMgr // Create ipset for the namespace. - if err = ipsMgr.CreateSet(nsName, util.IpsetNetHashFlag); err != nil { + if err = ipsMgr.CreateSet(nsName, append([]string{util.IpsetNetHashFlag})); err != nil { log.Errorf("Error: failed to create ipset for namespace %s.", nsName) return err } diff --git a/npm/npm.go b/npm/npm.go index 17075b12f0..4aa96a6be5 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -252,7 +252,7 @@ func NewNetworkPolicyManager(clientset *kubernetes.Clientset, informerFactory in // Create ipset for the namespace. kubeSystemNs := "ns-" + util.KubeSystemFlag - if err := allNs.ipsMgr.CreateSet(kubeSystemNs, util.IpsetNetHashFlag); err != nil { + if err := allNs.ipsMgr.CreateSet(kubeSystemNs, append([]string{util.IpsetNetHashFlag})); err != nil { log.Logf("Error: failed to create ipset for namespace %s.", kubeSystemNs) } diff --git a/npm/nwpolicy.go b/npm/nwpolicy.go index bf12fc273f..bf642efd64 100644 --- a/npm/nwpolicy.go +++ b/npm/nwpolicy.go @@ -3,8 +3,11 @@ package npm import ( + "strconv" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/npm/iptm" + "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/util" networkingv1 "k8s.io/api/networking/v1" ) @@ -49,7 +52,7 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP } if !npMgr.isAzureNpmChainCreated { - if err = allNs.ipsMgr.CreateSet(util.KubeSystemFlag, util.IpsetNetHashFlag); err != nil { + if err = allNs.ipsMgr.CreateSet(util.KubeSystemFlag, append([]string{util.IpsetNetHashFlag})); err != nil { log.Errorf("Error: failed to initialize kube-system ipset.") return err } @@ -63,11 +66,12 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP } var ( - hashedSelector = HashSelector(&npObj.Spec.PodSelector) - addedPolicy *networkingv1.NetworkPolicy - sets, namedPorts, lists []string - iptEntries []*iptm.IptEntry - ipsMgr = allNs.ipsMgr + hashedSelector = HashSelector(&npObj.Spec.PodSelector) + addedPolicy *networkingv1.NetworkPolicy + sets, namedPorts, lists []string + ingressIPCidrs, egressIPCidrs [][]string + iptEntries []*iptm.IptEntry + ipsMgr = allNs.ipsMgr ) // Remove the existing policy from processed (merged) network policy map @@ -93,16 +97,16 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP ns.rawNpMap[npObj.ObjectMeta.Name] = npObj - sets, namedPorts, lists, iptEntries = translatePolicy(npObj) + sets, namedPorts, lists, ingressIPCidrs, egressIPCidrs, iptEntries = translatePolicy(npObj) for _, set := range sets { log.Printf("Creating set: %v, hashedSet: %v", set, util.GetHashedName(set)) - if err = ipsMgr.CreateSet(set, util.IpsetNetHashFlag); err != nil { + if err = ipsMgr.CreateSet(set, append([]string{util.IpsetNetHashFlag})); err != nil { log.Printf("Error creating ipset %s", set) } } for _, set := range namedPorts { log.Printf("Creating set: %v, hashedSet: %v", set, util.GetHashedName(set)) - if err = ipsMgr.CreateSet(set, util.IpsetIPPortHashFlag); err != nil { + if err = ipsMgr.CreateSet(set, append([]string{util.IpsetIPPortHashFlag})); err != nil { log.Printf("Error creating ipset named port %s", set) } } @@ -114,6 +118,8 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP if err = npMgr.InitAllNsList(); err != nil { log.Printf("Error initializing all-namespace ipset list.") } + createCidrsRule("in", npObj.ObjectMeta.Name, ingressIPCidrs, ipsMgr) + createCidrsRule("out", npObj.ObjectMeta.Name, egressIPCidrs, ipsMgr) iptMgr := allNs.iptMgr for _, iptEntry := range iptEntries { if err = iptMgr.Add(iptEntry); err != nil { @@ -154,7 +160,7 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo npMgr.nsMap[npNs] = ns } - _, _, _, iptEntries := translatePolicy(npObj) + _, _, _, ingressIPCidrs, egressIPCidrs, iptEntries := translatePolicy(npObj) iptMgr := allNs.iptMgr for _, iptEntry := range iptEntries { @@ -163,6 +169,9 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo } } + removeCidrsRule("in", npObj.ObjectMeta.Name, ingressIPCidrs, allNs.ipsMgr) + removeCidrsRule("out", npObj.ObjectMeta.Name, egressIPCidrs, allNs.ipsMgr) + delete(ns.rawNpMap, npObj.ObjectMeta.Name) hashedSelector := HashSelector(&npObj.Spec.PodSelector) @@ -189,3 +198,35 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo return nil } + +func createCidrsRule(ingressOrEgress string, policyName string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { + spec := append([]string{util.IpsetNetHashFlag, util.IpsetMaxelemName, util.IpsetMaxelemNum}) + for i, ipCidrSet := range ipsetEntries { + if ipCidrSet == nil || len(ipCidrSet) == 0 { + continue + } + setName := policyName + "-cidr" + strconv.Itoa(i) + ingressOrEgress + log.Printf("Creating set: %v, hashedSet: %v", setName, util.GetHashedName(setName)) + if err := ipsMgr.CreateSet(setName, spec); err != nil { + log.Printf("Error creating ipset %s", ipCidrSet) + } + for _, ipCidrEntry := range util.DropEmptyFields(ipCidrSet) { + if err := ipsMgr.AddToSet(setName, ipCidrEntry, util.IpsetNetHashFlag); err != nil { + log.Printf("Error adding ip cidrs %s into ipset %s", ipCidrEntry, ipCidrSet) + } + } + } +} + +func removeCidrsRule(ingressOrEgress string, policyName string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { + for i, ipCidrSet := range ipsetEntries { + if ipCidrSet == nil || len(ipCidrSet) == 0 { + continue + } + setName := policyName + "-cidr" + strconv.Itoa(i) + ingressOrEgress + log.Printf("Delete set: %v, hashedSet: %v", setName, util.GetHashedName(setName)) + if err := ipsMgr.DeleteSet(setName); err != nil { + log.Printf("Error deleting ipset %s", ipCidrSet) + } + } +} \ No newline at end of file diff --git a/npm/nwpolicy_test.go b/npm/nwpolicy_test.go index 81a70f06bb..892fa71fb7 100644 --- a/npm/nwpolicy_test.go +++ b/npm/nwpolicy_test.go @@ -38,7 +38,7 @@ func TestAddNetworkPolicy(t *testing.T) { } // Create ns-kube-system set - if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestAddNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err) } @@ -161,7 +161,7 @@ func TestUpdateNetworkPolicy(t *testing.T) { }() // Create ns-kube-system set - if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestUpdateNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err) } @@ -273,7 +273,7 @@ func TestDeleteNetworkPolicy(t *testing.T) { }() // Create ns-kube-system set - if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, util.IpsetNetHashFlag); err != nil { + if err := ipsMgr.CreateSet("ns-"+util.KubeSystemFlag, append([]string{util.IpsetNetHashFlag})); err != nil { t.Errorf("TestDeleteNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err) } diff --git a/npm/pod.go b/npm/pod.go index e0a1e42f6e..a020b557f6 100644 --- a/npm/pod.go +++ b/npm/pod.go @@ -55,7 +55,7 @@ func (npMgr *NetworkPolicyManager) AddPod(podObj *corev1.Pod) error { // Add pod namespace if it doesn't exist if _, exists := npMgr.nsMap[podNs]; !exists { log.Printf("Creating set: %v, hashedSet: %v", podNs, util.GetHashedName(podNs)) - if err = ipsMgr.CreateSet(podNs, util.IpsetNetHashFlag); err != nil { + if err = ipsMgr.CreateSet(podNs, append([]string{util.IpsetNetHashFlag})); err != nil { log.Printf("Error creating ipset %s", podNs) } } diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go index aad03a8732..85cbf74633 100644 --- a/npm/translatePolicy.go +++ b/npm/translatePolicy.go @@ -3,6 +3,8 @@ package npm import ( + "strconv" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/npm/iptm" "github.com/Azure/azure-container-networking/npm/util" @@ -149,11 +151,12 @@ func craftPartialIptablesCommentFromSelector(ns string, selector *metav1.LabelSe return comment[:len(comment)-len("-AND-")] + postfix } -func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyIngressRule) ([]string, []string, []string, []*iptm.IptEntry) { +func translateIngress(ns string, name string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyIngressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { var ( sets []string // ipsets with type: net:hash namedPorts []string // ipsets with type: hash:ip,port lists []string // ipsets with type: list:set + ipCidrs [][]string entries []*iptm.IptEntry fromRuleEntries []*iptm.IptEntry addedIngressFromEntry, addedPortEntry bool // add drop entries at the end of the chain when there are non ALLOW-ALL* rules @@ -165,15 +168,17 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne ops, labels := GetOperatorsAndLabels(labelsWithOps) sets = append(sets, labels...) sets = append(sets, "ns-"+ns) + ipCidrs = make([][]string, len(rules)) targetSelectorIptEntrySpec := craftPartialIptEntrySpecFromOpsAndLabels(ns, ops, labels, util.IptablesDstFlag, false) targetSelectorComment := craftPartialIptablesCommentFromSelector(ns, &targetSelector, false) - for _, rule := range rules { + for i, rule := range rules { allowExternal := false portRuleExists := rule.Ports != nil && len(rule.Ports) > 0 fromRuleExists := false addedPortEntry = addedPortEntry || portRuleExists + ipCidrs[i] = make([]string, len(rule.From)) if rule.From != nil { if len(rule.From) == 0 { @@ -273,10 +278,21 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne } // fromRuleExists - for _, fromRule := range rule.From { + for j, fromRule := range rule.From { // Handle IPBlock field of NetworkPolicyPeer if fromRule.IPBlock != nil { if len(fromRule.IPBlock.CIDR) > 0 { + ipCidrs[i] = append(ipCidrs[i], fromRule.IPBlock.CIDR) + cidrIpsetName := name + "-cidr" + strconv.Itoa(i) + "in" + if len(fromRule.IPBlock.Except) > 0 { + for _, except := range fromRule.IPBlock.Except { + ipCidrs[i] = append(ipCidrs[i], except + " " + util.IpsetNomatch) + } + addedIngressFromEntry = true + } + if j != 0 { + continue + } if portRuleExists { for _, portRule := range rule.Ports { if portRule.Port.IntValue() == 0 { @@ -288,8 +304,11 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne } entry.Specs = append( entry.Specs, - util.IptablesSFlag, - fromRule.IPBlock.CIDR, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesSrcFlag, ) entry.Specs = append( entry.Specs, @@ -303,7 +322,7 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+fromRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-:-"+craftPartialIptablesCommentFromPort(portRule, util.IptablesDstPortFlag)+ "-TO-"+targetSelectorComment, ) @@ -315,8 +334,11 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne } entry.Specs = append( entry.Specs, - util.IptablesSFlag, - fromRule.IPBlock.CIDR, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesSrcFlag, ) entry.Specs = append( entry.Specs, @@ -329,7 +351,7 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+fromRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-:-"+craftPartialIptablesCommentFromPort(portRule, util.IptablesDstPortFlag)+ "-TO-"+targetSelectorComment, ) @@ -343,47 +365,22 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne } cidrEntry.Specs = append( cidrEntry.Specs, - util.IptablesSFlag, - fromRule.IPBlock.CIDR, - ) - cidrEntry.Specs = append( - cidrEntry.Specs, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesSrcFlag, util.IptablesJumpFlag, util.IptablesAccept, util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+fromRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-TO-"+targetSelectorComment, ) fromRuleEntries = append(fromRuleEntries, cidrEntry) addedIngressFromEntry = true } - if len(fromRule.IPBlock.Except) > 0 { - for _, except := range fromRule.IPBlock.Except { - exceptEntry := &iptm.IptEntry{ - Chain: util.IptablesAzureIngressFromChain, - } - exceptEntry.Specs = append( - exceptEntry.Specs, - util.IptablesSFlag, - except, - ) - exceptEntry.Specs = append(exceptEntry.Specs, targetSelectorIptEntrySpec...) - exceptEntry.Specs = append( - exceptEntry.Specs, - util.IptablesJumpFlag, - util.IptablesDrop, - util.IptablesModuleFlag, - util.IptablesCommentModuleFlag, - util.IptablesCommentFlag, - "DROP-"+except+ - "-TO-"+targetSelectorComment, - ) - fromRuleEntries = append(fromRuleEntries, exceptEntry) - } - addedIngressFromEntry = true - } } continue } @@ -790,14 +787,15 @@ func translateIngress(ns string, targetSelector metav1.LabelSelector, rules []ne } log.Printf("finished parsing ingress rule") - return util.DropEmptyFields(sets), util.DropEmptyFields(namedPorts), util.DropEmptyFields(lists), entries + return util.DropEmptyFields(sets), util.DropEmptyFields(namedPorts), util.DropEmptyFields(lists), ipCidrs, entries } -func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyEgressRule) ([]string, []string, []string, []*iptm.IptEntry) { +func translateEgress(ns string, name string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyEgressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { var ( sets []string // ipsets with type: net:hash namedPorts []string // ipsets with type: hash:ip,port lists []string // ipsets with type: list:set + ipCidrs [][]string entries []*iptm.IptEntry toRuleEntries []*iptm.IptEntry addedEgressToEntry, addedPortEntry bool // add drop entry when there are non ALLOW-ALL* rules @@ -809,14 +807,16 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net ops, labels := GetOperatorsAndLabels(labelsWithOps) sets = append(sets, labels...) sets = append(sets, "ns-"+ns) + ipCidrs = make([][]string, len(rules)) targetSelectorIptEntrySpec := craftPartialIptEntrySpecFromOpsAndLabels(ns, ops, labels, util.IptablesSrcFlag, false) targetSelectorComment := craftPartialIptablesCommentFromSelector(ns, &targetSelector, false) - for _, rule := range rules { + for i, rule := range rules { allowExternal := false portRuleExists := rule.Ports != nil && len(rule.Ports) > 0 toRuleExists := false addedPortEntry = addedPortEntry || portRuleExists + ipCidrs[i] = make([]string, len(rule.To)) if rule.To != nil { if len(rule.To) == 0 { @@ -913,10 +913,21 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net } // toRuleExists - for _, toRule := range rule.To { + for j, toRule := range rule.To { // Handle IPBlock field of NetworkPolicyPeer if toRule.IPBlock != nil { if len(toRule.IPBlock.CIDR) > 0 { + ipCidrs[i] = append(ipCidrs[i], toRule.IPBlock.CIDR) + cidrIpsetName := name + "-cidr" + strconv.Itoa(i) + "out" + if len(toRule.IPBlock.Except) > 0 { + for _, except := range toRule.IPBlock.Except { + ipCidrs[i] = append(ipCidrs[i], except+util.IpsetNomatch) + } + addedEgressToEntry = true + } + if j != 0 { + continue + } if portRuleExists { for _, portRule := range rule.Ports { if portRule.Port.IntValue() == 0 { @@ -928,11 +939,11 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net } entry.Specs = append( entry.Specs, - util.IptablesDFlag, - toRule.IPBlock.CIDR, - ) - entry.Specs = append( - entry.Specs, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesDstFlag, util.IptablesModuleFlag, util.IptablesSetModuleFlag, util.IptablesMatchSetFlag, @@ -943,7 +954,7 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+toRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-:-"+craftPartialIptablesCommentFromPort(portRule, util.IptablesDstPortFlag)+ "-FROM-"+targetSelectorComment, ) @@ -959,17 +970,17 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net ) entry.Specs = append( entry.Specs, - util.IptablesDFlag, - toRule.IPBlock.CIDR, - ) - entry.Specs = append( - entry.Specs, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesDstFlag, util.IptablesJumpFlag, util.IptablesAccept, util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+toRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-:-"+craftPartialIptablesCommentFromPort(portRule, util.IptablesDstPortFlag)+ "-FROM-"+targetSelectorComment, ) @@ -982,12 +993,15 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net } cidrEntry.Specs = append( cidrEntry.Specs, - util.IptablesDFlag, - toRule.IPBlock.CIDR, + targetSelectorIptEntrySpec..., ) cidrEntry.Specs = append( cidrEntry.Specs, - targetSelectorIptEntrySpec..., + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(cidrIpsetName), + util.IptablesDstFlag, ) cidrEntry.Specs = append( cidrEntry.Specs, @@ -996,37 +1010,12 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net util.IptablesModuleFlag, util.IptablesCommentModuleFlag, util.IptablesCommentFlag, - "ALLOW-"+toRule.IPBlock.CIDR+ + "ALLOW-"+cidrIpsetName+ "-FROM-"+targetSelectorComment, ) toRuleEntries = append(toRuleEntries, cidrEntry) addedEgressToEntry = true } - if len(toRule.IPBlock.Except) > 0 { - for _, except := range toRule.IPBlock.Except { - exceptEntry := &iptm.IptEntry{ - Chain: util.IptablesAzureEgressToChain, - Specs: append([]string(nil), targetSelectorIptEntrySpec...), - } - exceptEntry.Specs = append( - exceptEntry.Specs, - util.IptablesDFlag, - except, - ) - exceptEntry.Specs = append( - exceptEntry.Specs, - util.IptablesJumpFlag, - util.IptablesDrop, - util.IptablesModuleFlag, - util.IptablesCommentModuleFlag, - util.IptablesCommentFlag, - "DROP-"+except+ - "-FROM-"+targetSelectorComment, - ) - toRuleEntries = append(toRuleEntries, exceptEntry) - } - addedEgressToEntry = true - } } continue } @@ -1434,7 +1423,7 @@ func translateEgress(ns string, targetSelector metav1.LabelSelector, rules []net } log.Printf("finished parsing egress rule") - return util.DropEmptyFields(sets), util.DropEmptyFields(namedPorts), util.DropEmptyFields(lists), entries + return util.DropEmptyFields(sets), util.DropEmptyFields(namedPorts), util.DropEmptyFields(lists), ipCidrs, entries } // Drop all non-whitelisted packets. @@ -1492,11 +1481,13 @@ func getDefaultDropEntries(ns string, targetSelector metav1.LabelSelector, hasIn // 1. ipset set names generated from all podSelectors // 2. ipset list names generated from all namespaceSelectors // 3. iptables entries generated from the input network policy object. -func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []string, []*iptm.IptEntry) { +func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []string, [][]string, [][]string, []*iptm.IptEntry) { var ( resultSets []string resultNamedPorts []string resultLists []string + resultIngressIPCidrs [][]string + resultEgressIPCidrs [][]string entries []*iptm.IptEntry hasIngress, hasEgress bool ) @@ -1514,33 +1505,37 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s }() npNs := npObj.ObjectMeta.Namespace + name := npObj.ObjectMeta.Name if len(npObj.Spec.PolicyTypes) == 0 { - ingressSets, ingressNamedPorts, ingressLists, ingressEntries := translateIngress(npNs, npObj.Spec.PodSelector, npObj.Spec.Ingress) + ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Ingress) resultSets = append(resultSets, ingressSets...) resultNamedPorts = append(resultNamedPorts, ingressNamedPorts...) resultLists = append(resultLists, ingressLists...) + resultIngressIPCidrs = ingressIPCidrs entries = append(entries, ingressEntries...) - egressSets, egressNamedPorts, egressLists, egressEntries := translateEgress(npNs, npObj.Spec.PodSelector, npObj.Spec.Egress) + egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Egress) resultSets = append(resultSets, egressSets...) resultNamedPorts = append(resultNamedPorts, egressNamedPorts...) resultLists = append(resultLists, egressLists...) + resultEgressIPCidrs = egressIPCidrs entries = append(entries, egressEntries...) hasIngress = len(ingressSets) > 0 hasEgress = len(egressSets) > 0 entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...) - return util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultNamedPorts), util.UniqueStrSlice(resultLists), entries + return util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultNamedPorts), util.UniqueStrSlice(resultLists), resultIngressIPCidrs, resultEgressIPCidrs, entries } for _, ptype := range npObj.Spec.PolicyTypes { if ptype == networkingv1.PolicyTypeIngress { - ingressSets, ingressNamedPorts, ingressLists, ingressEntries := translateIngress(npNs, npObj.Spec.PodSelector, npObj.Spec.Ingress) + ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Ingress) resultSets = append(resultSets, ingressSets...) resultNamedPorts = append(resultNamedPorts, ingressNamedPorts...) resultLists = append(resultLists, ingressLists...) + resultIngressIPCidrs = ingressIPCidrs entries = append(entries, ingressEntries...) if npObj.Spec.Ingress != nil && @@ -1554,10 +1549,11 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s } if ptype == networkingv1.PolicyTypeEgress { - egressSets, egressNamedPorts, egressLists, egressEntries := translateEgress(npNs, npObj.Spec.PodSelector, npObj.Spec.Egress) + egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Egress) resultSets = append(resultSets, egressSets...) resultNamedPorts = append(resultNamedPorts, egressNamedPorts...) resultLists = append(resultLists, egressLists...) + resultEgressIPCidrs = egressIPCidrs entries = append(entries, egressEntries...) if npObj.Spec.Egress != nil && @@ -1575,5 +1571,5 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s log.Printf("Translating Policy: %+v", npObj) resultSets, resultLists = util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultLists) - return resultSets, resultNamedPorts, resultLists, entries + return resultSets, resultNamedPorts, resultLists, resultIngressIPCidrs, resultEgressIPCidrs, entries } diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go index 991ba58677..df27d91254 100644 --- a/npm/translatePolicy_test.go +++ b/npm/translatePolicy_test.go @@ -547,7 +547,7 @@ func TestGetDefaultDropEntries(t *testing.T) { func TestTranslateIngress(t *testing.T) { ns := "testnamespace" - + name := "testnetworkpolicyname" targetSelector := metav1.LabelSelector{ MatchLabels: map[string]string{ "context": "dev", @@ -640,7 +640,7 @@ func TestTranslateIngress(t *testing.T) { } util.IsNewNwPolicyVerFlag = true - sets, _, lists, iptEntries := translateIngress(ns, targetSelector, rules) + sets, _, lists, _, iptEntries := translateIngress(ns, name, targetSelector, rules) expectedSets := []string{ "context:dev", "testNotIn:frontend", @@ -851,6 +851,7 @@ func TestTranslateIngress(t *testing.T) { func TestTranslateEgress(t *testing.T) { ns := "testnamespace" + name := "testnetworkpolicyname" targetSelector := metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -944,7 +945,7 @@ func TestTranslateEgress(t *testing.T) { } util.IsNewNwPolicyVerFlag = true - sets, _, lists, iptEntries := translateEgress(ns, targetSelector, rules) + sets, _, lists, _, iptEntries := translateEgress(ns, name, targetSelector, rules) expectedSets := []string{ "context:dev", "testNotIn:frontend", @@ -1172,7 +1173,7 @@ func TestDenyAllPolicy(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(denyAllPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(denyAllPolicy) expectedSets := []string{"ns-testnamespace"} if !reflect.DeepEqual(sets, expectedSets) { @@ -1204,7 +1205,7 @@ func TestAllowBackendToFrontend(t *testing.T) { if err != nil { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowBackendToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowBackendToFrontendPolicy) expectedSets := []string{ "app:backend", @@ -1342,7 +1343,7 @@ func TestAllowAllToAppFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -1403,7 +1404,7 @@ func TestDenyAllToAppFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(denyAllToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(denyAllToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -1439,7 +1440,7 @@ func TestNamespaceToFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowNsTestNamespaceToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowNsTestNamespaceToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -1571,7 +1572,7 @@ func TestAllowAllNamespacesToAppFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowAllNsToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowAllNsToFrontendPolicy) expectedSets := []string{ "app:frontend", "ns-testnamespace", @@ -1704,7 +1705,7 @@ func TestAllowNamespaceDevToAppFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowNsDevToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowNsDevToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -1853,7 +1854,7 @@ func TestAllowAllToK0AndK1AndAppFrontend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowAllToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowAllToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -2054,7 +2055,7 @@ func TestAllowNsDevAndAppBackendToAppFrontend(t *testing.T) { } util.IsNewNwPolicyVerFlag = true - sets, _, lists, iptEntries := translatePolicy(allowNsDevAndBackendToFrontendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowNsDevAndBackendToFrontendPolicy) expectedSets := []string{ "app:frontend", @@ -2195,7 +2196,7 @@ func TestAllowInternalAndExternal(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowInternalAndExternalPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowInternalAndExternalPolicy) expectedSets := []string{ "app:backdoor", @@ -2256,7 +2257,7 @@ func TestAllowBackendToFrontendPort8000(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowBackendToFrontendPort8000Policy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowBackendToFrontendPort8000Policy) expectedSets := []string{ "app:frontend", @@ -2373,7 +2374,7 @@ func TestAllowMultipleLabelsToMultipleLabels(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowCniOrCnsToK8sPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowCniOrCnsToK8sPolicy) expectedSets := []string{ "app:k8s", @@ -2581,7 +2582,7 @@ func TestDenyAllFromAppBackend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(denyAllFromBackendPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(denyAllFromBackendPolicy) expectedSets := []string{ "app:backend", @@ -2617,7 +2618,7 @@ func TestAllowAllFromAppBackend(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowAllEgress) + sets, _, lists, _, _, iptEntries := translatePolicy(allowAllEgress) expectedSets := []string{ "app:backend", @@ -2699,7 +2700,7 @@ func TestDenyAllFromNsUnsafe(t *testing.T) { if err != nil { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(denyAllFromNsUnsafePolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(denyAllFromNsUnsafePolicy) expectedSets := []string{ "ns-unsafe", @@ -2733,7 +2734,7 @@ func TestAllowAppFrontendToTCPPort53UDPPort53Policy(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(allowFrontendToTCPPort53UDPPort53Policy) + sets, _, lists, _, _, iptEntries := translatePolicy(allowFrontendToTCPPort53UDPPort53Policy) expectedSets := []string{ "app:frontend", @@ -2917,7 +2918,7 @@ func TestComplexPolicy(t *testing.T) { t.Fatal(err) } - sets, _, lists, iptEntries := translatePolicy(k8sExamplePolicy) + sets, _, lists, ingressIPCidrs, egressIPCidrs, iptEntries := translatePolicy(k8sExamplePolicy) expectedSets := []string{ "role:db", @@ -2939,6 +2940,26 @@ func TestComplexPolicy(t *testing.T) { t.Errorf("expectedLists: %v", expectedLists) } + expectedIngressIPCidrs := [][]string { + {"", "", "", "172.17.0.0/16", "172.17.1.0/24 nomatch"}, + } + + expectedEgressIPCidrs := [][]string { + {"", "10.0.0.0/24"}, + } + + if !reflect.DeepEqual(ingressIPCidrs, expectedIngressIPCidrs) { + t.Errorf("translatedPolicy failed @ k8s-example-policy ingress IP Cidrs comparison") + t.Errorf("ingress IP Cidrs: %v", ingressIPCidrs) + t.Errorf("expected ingress IP Cidrs: %v", expectedIngressIPCidrs) + } + + if !reflect.DeepEqual(egressIPCidrs, expectedEgressIPCidrs) { + t.Errorf("translatedPolicy failed @ k8s-example-policy egress IP Cidrs comparison") + t.Errorf("egress IP Cidrs: %v", egressIPCidrs) + t.Errorf("expected egress IP Cidrs: %v", expectedEgressIPCidrs) + } + expectedIptEntries := []*iptm.IptEntry{} nonKubeSystemEntries := []*iptm.IptEntry{ &iptm.IptEntry{ @@ -2954,41 +2975,17 @@ func TestComplexPolicy(t *testing.T) { util.IptablesMatchSetFlag, util.GetHashedName("role:db"), util.IptablesDstFlag, - util.IptablesSFlag, - "172.17.0.0/16", + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName("k8s-example-policy" + "cidr" + "0" + "in"), + util.IptablesSrcFlag, util.IptablesProtFlag, "TCP", util.IptablesDstPortFlag, "6379", util.IptablesJumpFlag, util.IptablesAccept, - util.IptablesModuleFlag, - util.IptablesCommentModuleFlag, - util.IptablesCommentFlag, - "ALLOW-172.17.0.0/16-:-TCP-PORT-6379-TO-role:db-IN-ns-default", - }, - }, - &iptm.IptEntry{ - Chain: util.IptablesAzureIngressFromChain, - Specs: []string{ - util.IptablesSFlag, - "172.17.1.0/24", - util.IptablesModuleFlag, - util.IptablesSetModuleFlag, - util.IptablesMatchSetFlag, - util.GetHashedName("ns-default"), - util.IptablesDstFlag, - util.IptablesModuleFlag, - util.IptablesSetModuleFlag, - util.IptablesMatchSetFlag, - util.GetHashedName("role:db"), - util.IptablesDstFlag, - util.IptablesJumpFlag, - util.IptablesDrop, - util.IptablesModuleFlag, - util.IptablesCommentModuleFlag, - util.IptablesCommentFlag, - "DROP-172.17.1.0/24-TO-role:db-IN-ns-default", }, }, &iptm.IptEntry{ @@ -3117,14 +3114,19 @@ func TestComplexPolicy(t *testing.T) { util.IptablesMatchSetFlag, util.GetHashedName("role:db"), util.IptablesSrcFlag, - util.IptablesDFlag, - "10.0.0.0/24", + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName("k8s-example-policy" + "cidr" + "0" + "out"), + util.IptablesDstFlag, + // util.IptablesDFlag, + // "10.0.0.0/24", util.IptablesJumpFlag, util.IptablesAccept, - util.IptablesModuleFlag, - util.IptablesCommentModuleFlag, - util.IptablesCommentFlag, - "ALLOW-10.0.0.0/24-:-TCP-PORT-5978-FROM-role:db-IN-ns-default", + // util.IptablesModuleFlag, + // util.IptablesCommentModuleFlag, + // util.IptablesCommentFlag, + // "ALLOW-10.0.0.0/24-:-TCP-PORT-5978-FROM-role:db-IN-ns-default", }, }, &iptm.IptEntry{ @@ -3294,7 +3296,7 @@ func TestDropPrecedenceOverAllow(t *testing.T) { }, } - sets, _, lists, iptEntries := translatePolicy(denyAllPolicy) + sets, _, lists, _, _, iptEntries := translatePolicy(denyAllPolicy) expectedSets := []string{ "ns-default", } @@ -3311,7 +3313,7 @@ func TestDropPrecedenceOverAllow(t *testing.T) { t.Errorf("expectedLists: %v", expectedLists) } - sets, _, lists, finalIptEntries := translatePolicy(allowToPodPolicy) + sets, _, lists, _, _, finalIptEntries := translatePolicy(allowToPodPolicy) expectedSets = []string{ "app:test", "testIn:pod-A", @@ -3649,7 +3651,7 @@ func TestNamedPorts(t *testing.T) { t.Fatal(err) } - sets, namedPorts, lists, iptEntries := translatePolicy(namedPortPolicy) + sets, namedPorts, lists, _, _, iptEntries := translatePolicy(namedPortPolicy) expectedSets := []string{ "app:server", diff --git a/npm/util/const.go b/npm/util/const.go index ffab039827..da53d9685d 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -102,6 +102,11 @@ const ( AzureNpmFlag string = "azure-npm" AzureNpmPrefix string = "azure-npm-" + + IpsetMaxelemName string = "maxelem" + IpsetMaxelemNum string = "4294967295" + + IpsetNomatch string = "nomatch" ) //NPM telemetry constants. From 15716a6ec04dea72d21d64b8d74c5d839ecbaf8f Mon Sep 17 00:00:00 2001 From: Shufang Date: Wed, 17 Jun 2020 07:54:43 -0700 Subject: [PATCH 2/3] Move Cidr translation from iptable to ipset. --- npm/nwpolicy.go | 16 ++++++++-------- npm/translatePolicy.go | 8 +++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/npm/nwpolicy.go b/npm/nwpolicy.go index bf642efd64..b444c05f4d 100644 --- a/npm/nwpolicy.go +++ b/npm/nwpolicy.go @@ -118,8 +118,8 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP if err = npMgr.InitAllNsList(); err != nil { log.Printf("Error initializing all-namespace ipset list.") } - createCidrsRule("in", npObj.ObjectMeta.Name, ingressIPCidrs, ipsMgr) - createCidrsRule("out", npObj.ObjectMeta.Name, egressIPCidrs, ipsMgr) + createCidrsRule("in", npObj.ObjectMeta.Name, npObj.ObjectMeta.Namespace, ingressIPCidrs, ipsMgr) + createCidrsRule("out", npObj.ObjectMeta.Name, npObj.ObjectMeta.Namespace, egressIPCidrs, ipsMgr) iptMgr := allNs.iptMgr for _, iptEntry := range iptEntries { if err = iptMgr.Add(iptEntry); err != nil { @@ -169,8 +169,8 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo } } - removeCidrsRule("in", npObj.ObjectMeta.Name, ingressIPCidrs, allNs.ipsMgr) - removeCidrsRule("out", npObj.ObjectMeta.Name, egressIPCidrs, allNs.ipsMgr) + removeCidrsRule("in", npObj.ObjectMeta.Name, npObj.ObjectMeta.Namespace, ingressIPCidrs, allNs.ipsMgr) + removeCidrsRule("out", npObj.ObjectMeta.Name, npObj.ObjectMeta.Namespace, egressIPCidrs, allNs.ipsMgr) delete(ns.rawNpMap, npObj.ObjectMeta.Name) @@ -199,13 +199,13 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo return nil } -func createCidrsRule(ingressOrEgress string, policyName string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { +func createCidrsRule(ingressOrEgress, policyName, ns string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { spec := append([]string{util.IpsetNetHashFlag, util.IpsetMaxelemName, util.IpsetMaxelemNum}) for i, ipCidrSet := range ipsetEntries { if ipCidrSet == nil || len(ipCidrSet) == 0 { continue } - setName := policyName + "-cidr" + strconv.Itoa(i) + ingressOrEgress + setName := policyName + "-in-ns-" + ns + "-" + strconv.Itoa(i) + ingressOrEgress log.Printf("Creating set: %v, hashedSet: %v", setName, util.GetHashedName(setName)) if err := ipsMgr.CreateSet(setName, spec); err != nil { log.Printf("Error creating ipset %s", ipCidrSet) @@ -218,12 +218,12 @@ func createCidrsRule(ingressOrEgress string, policyName string, ipsetEntries [][ } } -func removeCidrsRule(ingressOrEgress string, policyName string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { +func removeCidrsRule(ingressOrEgress, policyName, ns string, ipsetEntries [][]string, ipsMgr *ipsm.IpsetManager) { for i, ipCidrSet := range ipsetEntries { if ipCidrSet == nil || len(ipCidrSet) == 0 { continue } - setName := policyName + "-cidr" + strconv.Itoa(i) + ingressOrEgress + setName := policyName + "-in-ns-" + ns + "-" + strconv.Itoa(i) + ingressOrEgress log.Printf("Delete set: %v, hashedSet: %v", setName, util.GetHashedName(setName)) if err := ipsMgr.DeleteSet(setName); err != nil { log.Printf("Error deleting ipset %s", ipCidrSet) diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go index 85cbf74633..a30d364915 100644 --- a/npm/translatePolicy.go +++ b/npm/translatePolicy.go @@ -283,7 +283,7 @@ func translateIngress(ns string, name string, targetSelector metav1.LabelSelecto if fromRule.IPBlock != nil { if len(fromRule.IPBlock.CIDR) > 0 { ipCidrs[i] = append(ipCidrs[i], fromRule.IPBlock.CIDR) - cidrIpsetName := name + "-cidr" + strconv.Itoa(i) + "in" + cidrIpsetName := name + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "in" if len(fromRule.IPBlock.Except) > 0 { for _, except := range fromRule.IPBlock.Except { ipCidrs[i] = append(ipCidrs[i], except + " " + util.IpsetNomatch) @@ -918,7 +918,7 @@ func translateEgress(ns string, name string, targetSelector metav1.LabelSelector if toRule.IPBlock != nil { if len(toRule.IPBlock.CIDR) > 0 { ipCidrs[i] = append(ipCidrs[i], toRule.IPBlock.CIDR) - cidrIpsetName := name + "-cidr" + strconv.Itoa(i) + "out" + cidrIpsetName := name + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "out" if len(toRule.IPBlock.Except) > 0 { for _, except := range toRule.IPBlock.Except { ipCidrs[i] = append(ipCidrs[i], except+util.IpsetNomatch) @@ -1512,21 +1512,19 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s resultSets = append(resultSets, ingressSets...) resultNamedPorts = append(resultNamedPorts, ingressNamedPorts...) resultLists = append(resultLists, ingressLists...) - resultIngressIPCidrs = ingressIPCidrs entries = append(entries, ingressEntries...) egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Egress) resultSets = append(resultSets, egressSets...) resultNamedPorts = append(resultNamedPorts, egressNamedPorts...) resultLists = append(resultLists, egressLists...) - resultEgressIPCidrs = egressIPCidrs entries = append(entries, egressEntries...) hasIngress = len(ingressSets) > 0 hasEgress = len(egressSets) > 0 entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...) - return util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultNamedPorts), util.UniqueStrSlice(resultLists), resultIngressIPCidrs, resultEgressIPCidrs, entries + return util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultNamedPorts), util.UniqueStrSlice(resultLists), ingressIPCidrs, egressIPCidrs, entries } for _, ptype := range npObj.Spec.PolicyTypes { From 46bb100fa71a8e9b2623b54be5f38be8d5eb1114 Mon Sep 17 00:00:00 2001 From: Shufang Date: Wed, 17 Jun 2020 22:16:29 -0700 Subject: [PATCH 3/3] Move Cidr translation from iptable to ipset. --- npm/ipsm/ipsm.go | 15 ++++++--- npm/ipsm/ipsm_test.go | 4 +++ npm/testpolicies/complex-policy.yaml | 2 ++ npm/translatePolicy.go | 28 +++++++++++------ npm/translatePolicy_test.go | 46 ++++++++++++++++++++++------ 5 files changed, 71 insertions(+), 24 deletions(-) diff --git a/npm/ipsm/ipsm.go b/npm/ipsm/ipsm.go index 6f4c9df183..6b61ae6310 100644 --- a/npm/ipsm/ipsm.go +++ b/npm/ipsm/ipsm.go @@ -81,7 +81,7 @@ func (ipsMgr *IpsetManager) CreateList(listName string) error { name: listName, operationFlag: util.IpsetCreationFlag, set: util.GetHashedName(listName), - spec: append([]string{util.IpsetSetListFlag}), + spec: []string{util.IpsetSetListFlag}, } log.Printf("Creating List: %+v", entry) if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { @@ -132,7 +132,7 @@ func (ipsMgr *IpsetManager) AddToList(listName string, setName string) error { entry := &ipsEntry{ operationFlag: util.IpsetAppendFlag, set: util.GetHashedName(listName), - spec: append([]string{util.GetHashedName(setName)}), + spec: []string{util.GetHashedName(setName)}, } if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { @@ -162,7 +162,7 @@ func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) erro entry := &ipsEntry{ operationFlag: util.IpsetDeletionFlag, set: hashedListName, - spec: append([]string{hashedSetName}), + spec: []string{hashedSetName}, } if _, err := ipsMgr.Run(entry); err != nil { @@ -239,11 +239,18 @@ func (ipsMgr *IpsetManager) AddToSet(setName, ip, spec string) error { if err := ipsMgr.CreateSet(setName, append([]string{util.IpsetNetHashFlag})); err != nil { return err } + var resultSpec []string + if strings.Contains(ip, util.IpsetNomatch) { + ip = strings.Trim(ip, util.IpsetNomatch) + resultSpec = append([]string{ip, util.IpsetNomatch}) + } else { + resultSpec = append([]string{ip}) + } entry := &ipsEntry{ operationFlag: util.IpsetAppendFlag, set: util.GetHashedName(setName), - spec: append([]string{ip}), + spec: resultSpec, } if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 { diff --git a/npm/ipsm/ipsm_test.go b/npm/ipsm/ipsm_test.go index 7bf968c886..1db3b9b846 100644 --- a/npm/ipsm/ipsm_test.go +++ b/npm/ipsm/ipsm_test.go @@ -173,6 +173,10 @@ func TestAddToSet(t *testing.T) { if err := ipsMgr.AddToSet("test-set", "1.2.3.4", util.IpsetNetHashFlag); err != nil { t.Errorf("TestAddToSet failed @ ipsMgr.AddToSet") } + + if err := ipsMgr.AddToSet("test-set", "1.2.3.4/nomatch", util.IpsetNetHashFlag); err != nil { + t.Errorf("TestAddToSet with nomatch failed @ ipsMgr.AddToSet") + } } func TestDeleteFromSet(t *testing.T) { diff --git a/npm/testpolicies/complex-policy.yaml b/npm/testpolicies/complex-policy.yaml index 057225e51c..58b6acf7ea 100644 --- a/npm/testpolicies/complex-policy.yaml +++ b/npm/testpolicies/complex-policy.yaml @@ -29,6 +29,8 @@ spec: - to: - ipBlock: cidr: 10.0.0.0/24 + except: + - 10.0.0.1/32 ports: - protocol: TCP port: 5978 diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go index a30d364915..d97a84fc47 100644 --- a/npm/translatePolicy.go +++ b/npm/translatePolicy.go @@ -151,7 +151,7 @@ func craftPartialIptablesCommentFromSelector(ns string, selector *metav1.LabelSe return comment[:len(comment)-len("-AND-")] + postfix } -func translateIngress(ns string, name string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyIngressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { +func translateIngress(ns string, policyName string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyIngressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { var ( sets []string // ipsets with type: net:hash namedPorts []string // ipsets with type: hash:ip,port @@ -283,10 +283,11 @@ func translateIngress(ns string, name string, targetSelector metav1.LabelSelecto if fromRule.IPBlock != nil { if len(fromRule.IPBlock.CIDR) > 0 { ipCidrs[i] = append(ipCidrs[i], fromRule.IPBlock.CIDR) - cidrIpsetName := name + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "in" + cidrIpsetName := policyName + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "in" if len(fromRule.IPBlock.Except) > 0 { for _, except := range fromRule.IPBlock.Except { - ipCidrs[i] = append(ipCidrs[i], except + " " + util.IpsetNomatch) + // TODO move IP cidrs rule to allow based only + ipCidrs[i] = append(ipCidrs[i], except + util.IpsetNomatch) } addedIngressFromEntry = true } @@ -790,7 +791,7 @@ func translateIngress(ns string, name string, targetSelector metav1.LabelSelecto return util.DropEmptyFields(sets), util.DropEmptyFields(namedPorts), util.DropEmptyFields(lists), ipCidrs, entries } -func translateEgress(ns string, name string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyEgressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { +func translateEgress(ns string, policyName string, targetSelector metav1.LabelSelector, rules []networkingv1.NetworkPolicyEgressRule) ([]string, []string, []string, [][]string, []*iptm.IptEntry) { var ( sets []string // ipsets with type: net:hash namedPorts []string // ipsets with type: hash:ip,port @@ -918,9 +919,10 @@ func translateEgress(ns string, name string, targetSelector metav1.LabelSelector if toRule.IPBlock != nil { if len(toRule.IPBlock.CIDR) > 0 { ipCidrs[i] = append(ipCidrs[i], toRule.IPBlock.CIDR) - cidrIpsetName := name + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "out" + cidrIpsetName := policyName + "-in-ns-" + ns + "-" + strconv.Itoa(i) + "out" if len(toRule.IPBlock.Except) > 0 { for _, except := range toRule.IPBlock.Except { + // TODO move IP cidrs rule to allow based only ipCidrs[i] = append(ipCidrs[i], except+util.IpsetNomatch) } addedEgressToEntry = true @@ -944,6 +946,9 @@ func translateEgress(ns string, name string, targetSelector metav1.LabelSelector util.IptablesMatchSetFlag, util.GetHashedName(cidrIpsetName), util.IptablesDstFlag, + ) + entry.Specs = append( + entry.Specs, util.IptablesModuleFlag, util.IptablesSetModuleFlag, util.IptablesMatchSetFlag, @@ -975,6 +980,9 @@ func translateEgress(ns string, name string, targetSelector metav1.LabelSelector util.IptablesMatchSetFlag, util.GetHashedName(cidrIpsetName), util.IptablesDstFlag, + ) + entry.Specs = append( + entry.Specs, util.IptablesJumpFlag, util.IptablesAccept, util.IptablesModuleFlag, @@ -1505,16 +1513,16 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s }() npNs := npObj.ObjectMeta.Namespace - name := npObj.ObjectMeta.Name + policyName := npObj.ObjectMeta.Name if len(npObj.Spec.PolicyTypes) == 0 { - ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Ingress) + ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, policyName, npObj.Spec.PodSelector, npObj.Spec.Ingress) resultSets = append(resultSets, ingressSets...) resultNamedPorts = append(resultNamedPorts, ingressNamedPorts...) resultLists = append(resultLists, ingressLists...) entries = append(entries, ingressEntries...) - egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Egress) + egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, policyName, npObj.Spec.PodSelector, npObj.Spec.Egress) resultSets = append(resultSets, egressSets...) resultNamedPorts = append(resultNamedPorts, egressNamedPorts...) resultLists = append(resultLists, egressLists...) @@ -1529,7 +1537,7 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s for _, ptype := range npObj.Spec.PolicyTypes { if ptype == networkingv1.PolicyTypeIngress { - ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Ingress) + ingressSets, ingressNamedPorts, ingressLists, ingressIPCidrs, ingressEntries := translateIngress(npNs, policyName, npObj.Spec.PodSelector, npObj.Spec.Ingress) resultSets = append(resultSets, ingressSets...) resultNamedPorts = append(resultNamedPorts, ingressNamedPorts...) resultLists = append(resultLists, ingressLists...) @@ -1547,7 +1555,7 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []s } if ptype == networkingv1.PolicyTypeEgress { - egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, name, npObj.Spec.PodSelector, npObj.Spec.Egress) + egressSets, egressNamedPorts, egressLists, egressIPCidrs, egressEntries := translateEgress(npNs, policyName, npObj.Spec.PodSelector, npObj.Spec.Egress) resultSets = append(resultSets, egressSets...) resultNamedPorts = append(resultNamedPorts, egressNamedPorts...) resultLists = append(resultLists, egressLists...) diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go index df27d91254..3d191ef78e 100644 --- a/npm/translatePolicy_test.go +++ b/npm/translatePolicy_test.go @@ -2941,11 +2941,11 @@ func TestComplexPolicy(t *testing.T) { } expectedIngressIPCidrs := [][]string { - {"", "", "", "172.17.0.0/16", "172.17.1.0/24 nomatch"}, + {"", "", "", "172.17.0.0/16", "172.17.1.0/24nomatch"}, } expectedEgressIPCidrs := [][]string { - {"", "10.0.0.0/24"}, + {"", "10.0.0.0/24", "10.0.0.1/32nomatch"}, } if !reflect.DeepEqual(ingressIPCidrs, expectedIngressIPCidrs) { @@ -2960,6 +2960,8 @@ func TestComplexPolicy(t *testing.T) { t.Errorf("expected egress IP Cidrs: %v", expectedEgressIPCidrs) } + cidrIngressIpsetName := "k8s-example-policy" + "-in-ns-" + "default-" + "0" + "in" + cidrEgressIpsetName := "k8s-example-policy" + "-in-ns-" + "default-" + "0" + "out" expectedIptEntries := []*iptm.IptEntry{} nonKubeSystemEntries := []*iptm.IptEntry{ &iptm.IptEntry{ @@ -2978,7 +2980,7 @@ func TestComplexPolicy(t *testing.T) { util.IptablesModuleFlag, util.IptablesSetModuleFlag, util.IptablesMatchSetFlag, - util.GetHashedName("k8s-example-policy" + "cidr" + "0" + "in"), + util.GetHashedName(cidrIngressIpsetName), util.IptablesSrcFlag, util.IptablesProtFlag, "TCP", @@ -2986,6 +2988,10 @@ func TestComplexPolicy(t *testing.T) { "6379", util.IptablesJumpFlag, util.IptablesAccept, + util.IptablesModuleFlag, + util.IptablesCommentModuleFlag, + util.IptablesCommentFlag, + "ALLOW-" + cidrIngressIpsetName + "-:-TCP-PORT-6379-TO-role:db-IN-ns-default", }, }, &iptm.IptEntry{ @@ -3117,21 +3123,41 @@ func TestComplexPolicy(t *testing.T) { util.IptablesModuleFlag, util.IptablesSetModuleFlag, util.IptablesMatchSetFlag, - util.GetHashedName("k8s-example-policy" + "cidr" + "0" + "out"), + util.GetHashedName(cidrEgressIpsetName), util.IptablesDstFlag, - // util.IptablesDFlag, - // "10.0.0.0/24", util.IptablesJumpFlag, util.IptablesAccept, - // util.IptablesModuleFlag, - // util.IptablesCommentModuleFlag, - // util.IptablesCommentFlag, - // "ALLOW-10.0.0.0/24-:-TCP-PORT-5978-FROM-role:db-IN-ns-default", + util.IptablesModuleFlag, + util.IptablesCommentModuleFlag, + util.IptablesCommentFlag, + "ALLOW-" + cidrEgressIpsetName + "-:-TCP-PORT-5978-FROM-role:db-IN-ns-default", }, }, &iptm.IptEntry{ Chain: util.IptablesAzureEgressPortChain, IsJumpEntry: true, + Specs: []string{ + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName("ns-default"), + util.IptablesSrcFlag, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName("role:db"), + util.IptablesSrcFlag, + util.IptablesJumpFlag, + util.IptablesAzureEgressToChain, + util.IptablesModuleFlag, + util.IptablesCommentModuleFlag, + util.IptablesCommentFlag, + "ALLOW-ALL-FROM-role:db-IN-ns-default-TO-JUMP-TO-" + util.IptablesAzureEgressToChain, + }, + }, + &iptm.IptEntry{ + Chain: util.IptablesAzureEgressToChain, + IsJumpEntry: true, Specs: []string{ util.IptablesModuleFlag, util.IptablesSetModuleFlag,