diff --git a/npm/npm_test.go b/npm/npm_test.go index 918b968aa2..e195421db7 100644 --- a/npm/npm_test.go +++ b/npm/npm_test.go @@ -9,7 +9,6 @@ import ( "github.com/Azure/azure-container-networking/npm/metrics" "k8s.io/client-go/tools/cache" "k8s.io/utils/exec" - utilexec "k8s.io/utils/exec" ) // To indicate the object is needed to be DeletedFinalStateUnknown Object @@ -29,15 +28,6 @@ func getKey(obj interface{}, t *testing.T) string { return key } -func newNPMgr(t *testing.T, exec utilexec.Interface) *NetworkPolicyManager { - npMgr := &NetworkPolicyManager{ - ipsMgr: ipsm.NewIpsetManager(exec), - TelemetryEnabled: false, - } - - return npMgr -} - func TestMain(m *testing.M) { metrics.InitializeAll() exec := exec.New() diff --git a/npm/pkg/dataplane/dataplane.go b/npm/pkg/dataplane/dataplane.go new file mode 100644 index 0000000000..ac7ea5ca9b --- /dev/null +++ b/npm/pkg/dataplane/dataplane.go @@ -0,0 +1,162 @@ +package dataplane + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/npm" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/policies" +) + +type DataPlane struct { + policyMgr policies.PolicyManager + ipsetMgr ipsets.IPSetManager + networkID string + // key is PodKey + endpointCache map[string]*NPMEndpoint +} + +type NPMEndpoint struct { + Name string + ID string + // Map with Key as Network Policy name to to emulate set + // and value as struct{} for minimal memory consumption + NetPolReference map[string]struct{} +} + +func NewDataPlane() *DataPlane { + return &DataPlane{ + policyMgr: policies.NewPolicyManager(), + ipsetMgr: ipsets.NewIPSetManager(), + endpointCache: make(map[string]*NPMEndpoint), + } +} + +// InitializeDataPlane helps in setting up dataplane for NPM +func (dp *DataPlane) InitializeDataPlane() error { + return dp.initializeDataPlane() +} + +// ResetDataPlane helps in cleaning up dataplane sets and policies programmed +// by NPM, retunring a clean slate +func (dp *DataPlane) ResetDataPlane() error { + return dp.resetDataPlane() +} + +// CreateIPSet takes in a set object and updates local cache with this set +func (dp *DataPlane) CreateIPSet(set *ipsets.IPSet) error { + err := dp.ipsetMgr.CreateIPSet(set) + if err != nil { + return fmt.Errorf("[DataPlane] error while creating set: %w", err) + } + return nil +} + +// DeleteSet checks for members and references of the given "set" type ipset +// if not used then will delete it from cache +func (dp *DataPlane) DeleteSet(name string) error { + err := dp.ipsetMgr.DeleteSet(name) + if err != nil { + return fmt.Errorf("[DataPlane] error while deleting set: %w", err) + } + return nil +} + +// DeleteList sanity checks and deletes a list ipset +func (dp *DataPlane) DeleteList(name string) error { + err := dp.ipsetMgr.DeleteList(name) + if err != nil { + return fmt.Errorf("[DataPlane] error while deleting list: %w", err) + } + return nil +} + +// AddToSet takes in a list of IPset objects along with IP member +// and then updates it local cache +func (dp *DataPlane) AddToSet(setNames []*ipsets.IPSet, ip, podKey string) error { + err := dp.ipsetMgr.AddToSet(setNames, ip, podKey) + if err != nil { + return fmt.Errorf("[DataPlane] error while adding to set: %w", err) + } + return nil +} + +// RemoveFromSet takes in list of setnames from which a given IP member should be +// removed and will update the local cache +func (dp *DataPlane) RemoveFromSet(setNames []string, ip, podKey string) error { + err := dp.ipsetMgr.RemoveFromSet(setNames, ip, podKey) + if err != nil { + return fmt.Errorf("[DataPlane] error while removing from set: %w", err) + } + return nil +} + +// AddToList takes a list name and list of sets which are to be added as members +// to given list +func (dp *DataPlane) AddToList(listName string, setNames []string) error { + err := dp.ipsetMgr.AddToList(listName, setNames) + if err != nil { + return fmt.Errorf("[DataPlane] error while adding to list: %w", err) + } + return nil +} + +// RemoveFromList takes a list name and list of sets which are to be removed as members +// to given list +func (dp *DataPlane) RemoveFromList(listName string, setNames []string) error { + err := dp.ipsetMgr.RemoveFromList(listName, setNames) + if err != nil { + return fmt.Errorf("[DataPlane] error while removing from list: %w", err) + } + return nil +} + +// UpdatePod is to be called by pod_controller ONLY when a new pod is CREATED. +func (dp *DataPlane) UpdatePod(pod *npm.NpmPod) error { + err := dp.updatePod(pod) + if err != nil { + return fmt.Errorf("[DataPlane] error while updating pod: %w", err) + } + return nil +} + +// ApplyDataPlane all the IPSet operations just update cache and update a dirty ipset structure, +// they do not change apply changes into dataplane. This function needs to be called at the +// end of IPSet operations of a given controller event, it will check for the dirty ipset list +// and accordingly makes changes in dataplane. This function helps emulate a single call to +// dataplane instead of multiple ipset operations calls ipset operations calls to dataplane +func (dp *DataPlane) ApplyDataPlane() error { + err := dp.ipsetMgr.ApplyIPSets(dp.networkID) + if err != nil { + return fmt.Errorf("[DataPlane] error while applying IPSets: %w", err) + } + return nil +} + +// AddPolicy takes in a translated NPMNetworkPolicy object and applies on dataplane +func (dp *DataPlane) AddPolicy(policies *policies.NPMNetworkPolicy) error { + err := dp.policyMgr.AddPolicy(policies) + if err != nil { + return fmt.Errorf("[DataPlane] error while adding policy: %w", err) + } + return nil +} + +// RemovePolicy takes in network policy name and removes it from dataplane and cache +func (dp *DataPlane) RemovePolicy(policyName string) error { + err := dp.policyMgr.RemovePolicy(policyName) + if err != nil { + return fmt.Errorf("[DataPlane] error while removing policy: %w", err) + } + return nil +} + +// UpdatePolicy takes in updated policy object, calculates the delta and applies changes +// onto dataplane accordingly +func (dp *DataPlane) UpdatePolicy(policies *policies.NPMNetworkPolicy) error { + err := dp.policyMgr.UpdatePolicy(policies) + if err != nil { + return fmt.Errorf("[DataPlane] error while updating policy: %w", err) + } + return nil +} diff --git a/npm/pkg/dataplane/dataplane_linux.go b/npm/pkg/dataplane/dataplane_linux.go new file mode 100644 index 0000000000..067028f09c --- /dev/null +++ b/npm/pkg/dataplane/dataplane_linux.go @@ -0,0 +1,21 @@ +package dataplane + +import ( + "github.com/Azure/azure-container-networking/npm" + "k8s.io/klog" +) + +// initializeDataPlane should be adding required chains and rules +func (dp *DataPlane) initializeDataPlane() error { + klog.Infof("Initializing dataplane for linux") + return nil +} + +// updatePod is no-op in Linux +func (dp *DataPlane) updatePod(pod *npm.NpmPod) error { + return nil +} + +func (dp *DataPlane) resetDataPlane() error { + return nil +} diff --git a/npm/pkg/dataplane/dataplane_test.go b/npm/pkg/dataplane/dataplane_test.go new file mode 100644 index 0000000000..5be84e2377 --- /dev/null +++ b/npm/pkg/dataplane/dataplane_test.go @@ -0,0 +1,23 @@ +package dataplane + +import ( + "testing" + + "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" +) + +func TestNewDataPlane(t *testing.T) { + metrics.InitializeAll() + dp := NewDataPlane() + + if dp == nil { + t.Error("NewDataPlane() returned nil") + } + set := ipsets.NewIPSet("test", ipsets.NameSpace) + + err := dp.CreateIPSet(set) + if err != nil { + t.Error("CreateIPSet() returned error") + } +} diff --git a/npm/pkg/dataplane/dataplane_windows.go b/npm/pkg/dataplane/dataplane_windows.go new file mode 100644 index 0000000000..051a79421a --- /dev/null +++ b/npm/pkg/dataplane/dataplane_windows.go @@ -0,0 +1,54 @@ +package dataplane + +import ( + "github.com/Azure/azure-container-networking/npm" + "github.com/Microsoft/hcsshim/hcn" + "k8s.io/klog" +) + +const ( + // Windows specific constants + AzureNetworkName = "azure" +) + +// initializeDataPlane will help gather network and endpoint details +func (dp *DataPlane) initializeDataPlane() error { + klog.Infof("Initializing dataplane for windows") + + // Get Network ID + network, err := hcn.GetNetworkByName(AzureNetworkName) + if err != nil { + return err + } + + dp.networkID = network.Id + + endpoints, err := hcn.ListEndpointsOfNetwork(dp.networkID) + if err != nil { + return err + } + + for _, endpoint := range endpoints { + klog.Infof("Endpoints info %+v", endpoint.Policies) + ep := &NPMEndpoint{ + Name: endpoint.Name, + ID: endpoint.Id, + NetPolReference: make(map[string]struct{}), + } + + dp.endpointCache[ep.Name] = ep + } + + return nil +} + +// updatePod has two responsibilities in windows +// 1. Will call into dataplane and updates endpoint references of this pod. +// 2. Will check for existing applicable network policies and applies it on endpoint +func (dp *DataPlane) updatePod(pod *npm.NpmPod) error { + return nil +} + +func (dp *DataPlane) resetDataPlane() error { + return nil +} diff --git a/npm/pkg/dataplane/ipsets/ipset.go b/npm/pkg/dataplane/ipsets/ipset.go new file mode 100644 index 0000000000..5b06ad340c --- /dev/null +++ b/npm/pkg/dataplane/ipsets/ipset.go @@ -0,0 +1,209 @@ +package ipsets + +import ( + "errors" + + "github.com/Azure/azure-container-networking/npm/util" +) + +type IPSet struct { + Name string + HashedName string + // SetProperties embedding set properties + SetProperties + // IpPodKey is used for setMaps to store Ips and ports as keys + // and podKey as value + IPPodKey map[string]string + // This is used for listMaps to store child IP Sets + MemberIPSets map[string]*IPSet + // Using a map to emulate set and value as struct{} for + // minimal memory consumption + // SelectorReference holds networkpolicy names where this IPSet + // is being used in PodSelector and NameSpace + SelectorReference map[string]struct{} + // NetPolReference holds networkpolicy names where this IPSet + // is being referred as part of rules + NetPolReference map[string]struct{} + // IpsetReferCount keeps count of 2nd level Nested IPSets + // with member as this IPSet + IpsetReferCount int +} + +type SetProperties struct { + // Stores type of ip grouping + Type SetType + // Stores kind of ipset in dataplane + Kind SetKind +} + +type SetType int8 + +const ( + // Unknown SetType + Unknown SetType = 0 + // NameSpace IPSet is created to hold + // ips of pods in a given NameSapce + NameSpace SetType = 1 + // KeyLabelOfNameSpace IPSet is a list kind ipset + // with members as ipsets of namespace with this Label Key + KeyLabelOfNameSpace SetType = 2 + // KeyValueLabelOfNameSpace IPSet is a list kind ipset + // with members as ipsets of namespace with this Label + KeyValueLabelOfNameSpace SetType = 3 + // KeyLabelOfPod IPSet contains IPs of Pods with this Label Key + KeyLabelOfPod SetType = 4 + // KeyValueLabelOfPod IPSet contains IPs of Pods with this Label + KeyValueLabelOfPod SetType = 5 + // NamedPorts IPSets contains a given namedport + NamedPorts SetType = 6 + // NestedLabelOfPod is derived for multivalue matchexpressions + NestedLabelOfPod SetType = 7 + // CIDRBlocks holds CIDR blocks + CIDRBlocks SetType = 8 +) + +var ( + setTypeName = map[SetType]string{ + Unknown: "Unknown", + NameSpace: "NameSpace", + KeyLabelOfNameSpace: "KeyLabelOfNameSpace", + KeyValueLabelOfNameSpace: "KeyValueLabelOfNameSpace", + KeyLabelOfPod: "KeyLabelOfPod", + KeyValueLabelOfPod: "KeyValueLabelOfPod", + NamedPorts: "NamedPorts", + NestedLabelOfPod: "NestedLabelOfPod", + CIDRBlocks: "CIDRBlocks", + } + // ErrIPSetInvalidKind is returned when IPSet kind is invalid + ErrIPSetInvalidKind = errors.New("Invalid IPSet Kind") +) + +func (x SetType) String() string { + return setTypeName[x] +} + +type SetKind string + +const ( + // ListSet is of kind list with members as other IPSets + ListSet SetKind = "list" + // HashSet is of kind hashset with members as IPs and/or port + HashSet SetKind = "set" +) + +func NewIPSet(name string, setType SetType) *IPSet { + set := &IPSet{ + Name: name, + HashedName: util.GetHashedName(name), + SetProperties: SetProperties{ + Type: setType, + Kind: getSetKind(setType), + }, + // Map with Key as Network Policy name to to emulate set + // and value as struct{} for minimal memory consumption + SelectorReference: make(map[string]struct{}), + // Map with Key as Network Policy name to to emulate set + // and value as struct{} for minimal memory consumption + NetPolReference: make(map[string]struct{}), + IpsetReferCount: 0, + } + if set.Kind == HashSet { + set.IPPodKey = make(map[string]string) + } else { + set.MemberIPSets = make(map[string]*IPSet) + } + return set +} + +func (set *IPSet) GetSetContents() ([]string, error) { + switch set.Kind { + case HashSet: + i := 0 + contents := make([]string, len(set.IPPodKey)) + for podIP := range set.IPPodKey { + contents[i] = podIP + i++ + } + return contents, nil + case ListSet: + i := 0 + contents := make([]string, len(set.MemberIPSets)) + for _, memberSet := range set.MemberIPSets { + contents[i] = memberSet.HashedName + i++ + } + return contents, nil + default: + return []string{}, ErrIPSetInvalidKind + } +} + +func getSetKind(setType SetType) SetKind { + switch setType { + case CIDRBlocks: + return HashSet + case NameSpace: + return HashSet + case NamedPorts: + return HashSet + case KeyLabelOfPod: + return HashSet + case KeyValueLabelOfPod: + return HashSet + case KeyLabelOfNameSpace: + return ListSet + case KeyValueLabelOfNameSpace: + return ListSet + case NestedLabelOfPod: + return ListSet + case Unknown: // adding this to appease golint + return "unknown" + default: + return "unknown" + } +} + +func (set *IPSet) AddMemberIPSet(memberIPSet *IPSet) { + set.MemberIPSets[memberIPSet.Name] = memberIPSet +} + +func (set *IPSet) IncIpsetReferCount() { + set.IpsetReferCount++ +} + +func (set *IPSet) DecIpsetReferCount() { + if set.IpsetReferCount == 0 { + return + } + set.IpsetReferCount-- +} + +func (set *IPSet) AddSelectorReference(netPolName string) { + set.SelectorReference[netPolName] = struct{}{} +} + +func (set *IPSet) DeleteSelectorReference(netPolName string) { + delete(set.SelectorReference, netPolName) +} + +func (set *IPSet) AddNetPolReference(netPolName string) { + set.NetPolReference[netPolName] = struct{}{} +} + +func (set *IPSet) DeleteNetPolReference(netPolName string) { + delete(set.NetPolReference, netPolName) +} + +func (set *IPSet) CanBeDeleted() bool { + return len(set.SelectorReference) == 0 && + len(set.NetPolReference) == 0 && + set.IpsetReferCount == 0 && + len(set.MemberIPSets) == 0 && + len(set.IPPodKey) == 0 +} + +// UsedByNetPol check if an IPSet is referred in network policies. +func (set *IPSet) UsedByNetPol() bool { + return len(set.SelectorReference) > 0 && + len(set.NetPolReference) > 0 +} diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go new file mode 100644 index 0000000000..6aa36ae163 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -0,0 +1,295 @@ +package ipsets + +import ( + "fmt" + "net" + "sync" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/npm/metrics" + npmerrors "github.com/Azure/azure-container-networking/npm/util/errors" +) + +type IPSetManager struct { + setMap map[string]*IPSet + // Map with Key as IPSet name to to emulate set + // and value as struct{} for minimal memory consumption + dirtyCaches map[string]struct{} + sync.Mutex +} + +func (iMgr *IPSetManager) exists(name string) bool { + _, ok := iMgr.setMap[name] + return ok +} + +func NewIPSetManager() IPSetManager { + return IPSetManager{ + setMap: make(map[string]*IPSet), + dirtyCaches: make(map[string]struct{}), + } +} + +func (iMgr *IPSetManager) updateDirtyCache(setName string) { + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return + } + + // If set is not referenced in netpol then ignore the update + if len(set.NetPolReference) == 0 && len(set.SelectorReference) == 0 { + return + } + + iMgr.dirtyCaches[set.Name] = struct{}{} + if set.Kind == ListSet { + // TODO check if we will need to add all the member ipsets + // also to the dirty cache list + for _, member := range set.MemberIPSets { + iMgr.dirtyCaches[member.Name] = struct{}{} + } + } +} + +func (iMgr *IPSetManager) clearDirtyCache() { + iMgr.dirtyCaches = make(map[string]struct{}) +} + +func (iMgr *IPSetManager) CreateIPSet(set *IPSet) error { + iMgr.Lock() + defer iMgr.Unlock() + return iMgr.createIPSet(set) +} + +func (iMgr *IPSetManager) createIPSet(set *IPSet) error { + // Check if the Set already exists + if iMgr.exists(set.Name) { + // ipset already exists + // we should calculate a diff if the members are different + return nil + } + + // append the cache if dataplane specific function + // return nil as error + iMgr.setMap[set.Name] = set + metrics.IncNumIPSets() + return nil +} + +func (iMgr *IPSetManager) AddToSet(addToSets []*IPSet, ip, podKey string) error { + // check if the IP is IPV4 family + if net.ParseIP(ip).To4() == nil { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, "IPV6 not supported") + } + iMgr.Lock() + defer iMgr.Unlock() + + for _, updatedSet := range addToSets { + set, exists := iMgr.setMap[updatedSet.Name] // check if the Set exists + if !exists { + err := iMgr.createIPSet(updatedSet) + if err != nil { + return err + } + set = iMgr.setMap[updatedSet.Name] + } + + if set.Kind != HashSet { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s is not a hash set", set.Name)) + } + cachedPodKey, ok := set.IPPodKey[ip] + if ok { + if cachedPodKey != podKey { + log.Logf("AddToSet: PodOwner has changed for Ip: %s, setName:%s, Old podKey: %s, new podKey: %s. Replace context with new PodOwner.", + ip, set.Name, cachedPodKey, podKey) + + set.IPPodKey[ip] = podKey + } + return nil + } + + // update the IP ownership with podkey + set.IPPodKey[ip] = podKey + iMgr.updateDirtyCache(set.Name) + + // Update metrics of the IpSet + metrics.AddEntryToIPSet(set.Name) + } + + return nil +} + +func (iMgr *IPSetManager) RemoveFromSet(removeFromSets []string, ip, podKey string) error { + iMgr.Lock() + defer iMgr.Unlock() + for _, setName := range removeFromSets { + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", setName)) + } + + if set.Kind != HashSet { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a hash set", setName)) + } + + // in case the IP belongs to a new Pod, then ignore this Delete call as this might be stale + cachedPodKey := set.IPPodKey[ip] + if cachedPodKey != podKey { + log.Logf("DeleteFromSet: PodOwner has changed for Ip: %s, setName:%s, Old podKey: %s, new podKey: %s. Ignore the delete as this is stale update", + ip, setName, cachedPodKey, podKey) + + return nil + } + + // update the IP ownership with podkey + delete(set.IPPodKey, ip) + iMgr.updateDirtyCache(set.Name) + + // Update metrics of the IpSet + metrics.RemoveEntryFromIPSet(setName) + } + + return nil +} + +func (iMgr *IPSetManager) AddToList(listName string, setNames []string) error { + iMgr.Lock() + defer iMgr.Unlock() + + for _, setName := range setNames { + if listName == setName { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("list %s cannot be added to itself", listName)) + } + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName)) + } + + // Nested IPSets are only supported for windows + // Check if we want to actually use that support + if set.Kind != HashSet { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("member ipset %s is not a Set type and nestetd ipsets are not supported", setName)) + } + + list, exists := iMgr.setMap[listName] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s does not exist", listName)) + } + + if list.Kind != ListSet { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s is not a list set", listName)) + } + + // check if Set is a member of List + listSet, exists := list.MemberIPSets[setName] + if exists { + if listSet == set { + // Set is already a member of List + return nil + } + // Update the ipset in list + list.MemberIPSets[setName] = set + return nil + } + + // update the Ipset member list of list + list.AddMemberIPSet(set) + set.IncIpsetReferCount() + // Update metrics of the IpSet + metrics.AddEntryToIPSet(listName) + } + + iMgr.updateDirtyCache(listName) + + return nil +} + +func (iMgr *IPSetManager) RemoveFromList(listName string, setNames []string) error { + iMgr.Lock() + defer iMgr.Unlock() + for _, setName := range setNames { + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", setName)) + } + + if set.Kind != HashSet { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a hash set", setName)) + } + + // Nested IPSets are only supported for windows + // Check if we want to actually use that support + if set.Kind != HashSet { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("member ipset %s is not a Set type and nestetd ipsets are not supported", setName)) + } + + list, exists := iMgr.setMap[listName] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", listName)) + } + + if list.Kind != ListSet { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a list set", listName)) + } + + // check if Set is a member of List + _, exists = list.MemberIPSets[setName] + if !exists { + return nil + } + + // delete IPSet from the list + delete(list.MemberIPSets, setName) + set.DecIpsetReferCount() + // Update metrics of the IpSet + metrics.RemoveEntryFromIPSet(listName) + } + iMgr.updateDirtyCache(listName) + + return nil +} + +func (iMgr *IPSetManager) DeleteList(name string) error { + iMgr.Lock() + defer iMgr.Unlock() + set, exists := iMgr.setMap[name] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", set.Name)) + } + + if !set.CanBeDeleted() { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s cannot be deleted", set.Name)) + } + + delete(iMgr.setMap, name) + return nil +} + +func (iMgr *IPSetManager) DeleteSet(name string) error { + iMgr.Lock() + defer iMgr.Unlock() + set, exists := iMgr.setMap[name] // check if the Set exists + if !exists { + return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", set.Name)) + } + + if !set.CanBeDeleted() { + return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s cannot be deleted", set.Name)) + } + delete(iMgr.setMap, name) + return nil +} + +func (iMgr *IPSetManager) ApplyIPSets(networkID string) error { + iMgr.Lock() + defer iMgr.Unlock() + + // Call the appropriate apply ipsets + err := iMgr.applyIPSets(networkID) + if err != nil { + return err + } + + iMgr.clearDirtyCache() + return nil +} diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go new file mode 100644 index 0000000000..f93d6d898a --- /dev/null +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -0,0 +1,21 @@ +package ipsets + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/npm/util/errors" + "k8s.io/klog" +) + +func (iMgr *IPSetManager) applyIPSets(networkID string) error { + for setName := range iMgr.dirtyCaches { + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName)) + } + + klog.Infof(set.Name) + + } + return nil +} diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go new file mode 100644 index 0000000000..9749e526e3 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go @@ -0,0 +1,151 @@ +package ipsets + +import ( + "fmt" + "os" + "testing" + + "github.com/Azure/azure-container-networking/npm/metrics" +) + +func TestCreateIPSet(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("Test", NameSpace) + + err := iMgr.CreateIPSet(set) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } +} + +func TestAddToSet(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("Test", NameSpace) + + fmt.Println(set.Name) + err := iMgr.AddToSet([]*IPSet{set}, "10.0.0.0", "test") + if err != nil { + t.Errorf("AddToSet() returned error %s", err.Error()) + } +} + +func TestRemoveFromSet(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("Test", NameSpace) + + err := iMgr.AddToSet([]*IPSet{set}, "10.0.0.0", "test") + if err != nil { + t.Errorf("RemoveFromSet() returned error %s", err.Error()) + } + err = iMgr.RemoveFromSet([]string{"Test"}, "10.0.0.0", "test") + if err != nil { + t.Errorf("RemoveFromSet() returned error %s", err.Error()) + } +} + +func TestRemoveFromSetMissing(t *testing.T) { + iMgr := NewIPSetManager() + err := iMgr.RemoveFromSet([]string{"Test"}, "10.0.0.0", "test") + if err == nil { + t.Errorf("RemoveFromSet() did not return error") + } +} + +func TestAddToListMissing(t *testing.T) { + iMgr := NewIPSetManager() + err := iMgr.AddToList("test", []string{"newtest"}) + if err == nil { + t.Errorf("AddToList() did not return error") + } +} + +func TestAddToList(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("newtest", NameSpace) + err := iMgr.CreateIPSet(set) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + list := NewIPSet("test", KeyLabelOfNameSpace) + err = iMgr.CreateIPSet(list) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + err = iMgr.AddToList("test", []string{"newtest"}) + if err != nil { + t.Errorf("AddToList() returned error %s", err.Error()) + } +} + +func TestRemoveFromList(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("newtest", NameSpace) + err := iMgr.CreateIPSet(set) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + list := NewIPSet("test", KeyLabelOfNameSpace) + err = iMgr.CreateIPSet(list) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + err = iMgr.AddToList("test", []string{"newtest"}) + if err != nil { + t.Errorf("AddToList() returned error %s", err.Error()) + } + + err = iMgr.RemoveFromList("test", []string{"newtest"}) + if err != nil { + t.Errorf("RemoveFromList() returned error %s", err.Error()) + } +} + +func TestRemoveFromListMissing(t *testing.T) { + iMgr := NewIPSetManager() + err := iMgr.RemoveFromList("test", []string{"newtest"}) + if err == nil { + t.Errorf("RemoveFromList() did not return error") + } +} + +func TestDeleteList(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("Test", KeyValueLabelOfNameSpace) + + err := iMgr.CreateIPSet(set) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + err = iMgr.DeleteList(set.Name) + if err != nil { + t.Errorf("DeleteList() returned error %s", err.Error()) + } +} + +func TestDeleteSet(t *testing.T) { + iMgr := NewIPSetManager() + set := NewIPSet("Test", NameSpace) + + err := iMgr.CreateIPSet(set) + if err != nil { + t.Errorf("CreateIPSet() returned error %s", err.Error()) + } + + err = iMgr.DeleteSet(set.Name) + if err != nil { + t.Errorf("DeleteSet() returned error %s", err.Error()) + } +} + +func TestMain(m *testing.M) { + metrics.InitializeAll() + + exitCode := m.Run() + + os.Exit(exitCode) +} diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go new file mode 100644 index 0000000000..9e3354f461 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -0,0 +1,181 @@ +package ipsets + +import ( + "encoding/json" + "fmt" + + "github.com/Azure/azure-container-networking/npm/util" + "github.com/Azure/azure-container-networking/npm/util/errors" + "github.com/Microsoft/hcsshim/hcn" + "k8s.io/klog" +) + +// SetPolicyTypes associated with SetPolicy. Value is IPSET. +type SetPolicyType string + +const ( + SetPolicyTypeIpSet SetPolicyType = "IPSET" + SetPolicyTypeNestedIpSet SetPolicyType = "NESTEDIPSET" +) + +// SetPolicySetting creates IPSets on network +type SetPolicySetting struct { + Id string + Name string + Type SetPolicyType + Values string +} + +func (iMgr *IPSetManager) applyIPSets(networkID string) error { + network, err := hcn.GetNetworkByID(networkID) + if err != nil { + return err + } + + setPolNames, err := getAllSetPolicyNames(network.Policies) + if err != nil { + return err + } + + setPolSettings, err := iMgr.calculateNewSetPolicies(setPolNames) + if err != nil { + return err + } + + policyNetworkRequest := hcn.PolicyNetworkRequest{ + Policies: []hcn.NetworkPolicy{}, + } + + for _, policy := range network.Policies { + // TODO (vamsi) use NetPolicyType constant setpolicy for below check + // after updating HCSShim + if policy.Type != "SetPolicy" { + policyNetworkRequest.Policies = append(policyNetworkRequest.Policies, policy) + } + } + + for setPol := range setPolSettings { + rawSettings, err := json.Marshal(setPolSettings[setPol]) + if err != nil { + return err + } + policyNetworkRequest.Policies = append( + policyNetworkRequest.Policies, + hcn.NetworkPolicy{ + Type: "SetPolicy", + Settings: rawSettings, + }, + ) + } + + err = network.AddPolicy(policyNetworkRequest) + if err != nil { + return err + } + + return nil +} + +func (iMgr *IPSetManager) calculateNewSetPolicies(existingSets []string) (map[string]SetPolicySetting, error) { + // some of this below logic can be abstracted a step above + dirtySets := iMgr.dirtyCaches + + for _, setName := range existingSets { + dirtySets[setName] = struct{}{} + } + + setsToUpdate := make(map[string]SetPolicySetting) + for setName := range dirtySets { + set, exists := iMgr.setMap[setName] // check if the Set exists + if !exists { + return nil, errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName)) + } + if !set.UsedByNetPol() { + continue + } + + setPol, err := convertToSetPolicy(set) + if err != nil { + return nil, err + } + setsToUpdate[setName] = setPol + if set.Kind == ListSet { + for _, memberSet := range set.MemberIPSets { + // TODO check whats the name here, hashed or normal + if _, ok := setsToUpdate[memberSet.Name]; ok { + continue + } + setPol, err = convertToSetPolicy(memberSet) + if err != nil { + return nil, err + } + setsToUpdate[memberSet.Name] = setPol + } + } + } + + return setsToUpdate, nil +} + +func isValidIPSet(set *IPSet) error { + if set.Name == "" { + return fmt.Errorf("IPSet " + set.Name + " is missing Name") + } + + if set.Type == Unknown { + return fmt.Errorf("IPSet " + set.Type.String() + " is missing Type") + } + + if set.HashedName == "" { + return fmt.Errorf("IPSet " + set.HashedName + " is missing HashedName") + } + + return nil +} + +func getSetPolicyType(set *IPSet) SetPolicyType { + switch set.Kind { + case ListSet: + return SetPolicyTypeNestedIpSet + case HashSet: + return SetPolicyTypeIpSet + default: + return "Unknown" + } +} + +func convertToSetPolicy(set *IPSet) (SetPolicySetting, error) { + err := isValidIPSet(set) + if err != nil { + return SetPolicySetting{}, err + } + + setContents, err := set.GetSetContents() + if err != nil { + return SetPolicySetting{}, err + } + + setPolicy := SetPolicySetting{ + Id: set.HashedName, + Name: set.Name, + Type: getSetPolicyType(set), + Values: util.SliceToString(setContents), + } + return setPolicy, nil +} + +func getAllSetPolicyNames(networkPolicies []hcn.NetworkPolicy) ([]string, error) { + setPols := []string{} + for _, netpol := range networkPolicies { + if netpol.Type == "SetPolicy" { + var set SetPolicySetting + err := json.Unmarshal(netpol.Settings, &set) + if err != nil { + klog.Error(err.Error()) + continue + } + setPols = append(setPols, set.Name) + } + } + return setPols, nil +} diff --git a/npm/pkg/dataplane/policies/policy.go b/npm/pkg/dataplane/policies/policy.go new file mode 100644 index 0000000000..064a1a0fee --- /dev/null +++ b/npm/pkg/dataplane/policies/policy.go @@ -0,0 +1,92 @@ +package policies + +import ( + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" + networkingv1 "k8s.io/api/networking/v1" +) + +type NPMNetworkPolicy struct { + Name string + // PodSelectorIPSets holds all the IPSets generated from Pod Selector + PodSelectorIPSets []*ipsets.IPSet + // RuleIPSets holds all IPSets generated from policy's rules + // and not from pod selector IPSets + RuleIPSets []*ipsets.IPSet + ACLs []*ACLPolicy + // Making this a podKey instead should be + // use NPMPod obj + Pods []string + RawNP *networkingv1.NetworkPolicy +} + +// ACLPolicy equivalent to a single iptable rule in linux +// or a single HNS rule in windows +type ACLPolicy struct { + // PolicyID is the rules name with a given network policy + PolicyID string + // Comment is the string attached to rule to identity its representation + Comment string + // SrcList source IPSets condition setinfos + SrcList []SetInfo + // DstList destination IPSets condition setinfos + DstList []SetInfo + // Target defines a target in iptables for linux. i,e, Mark, Accept, Drop + // in windows, this is either ALLOW or DENY + Target Verdict + // Direction defines the flow of traffic + Direction Direction + // SrcPorts holds the source port information + SrcPorts []Ports + // DstPorts holds the destination port information + DstPorts []Ports + // Protocol is the value of traffic protocol + Protocol Protocol +} + +// SetInfo helps capture additional details in a matchSet +// example match set in linux: +// ! azure-npm-123 src,src +// "!" this indicates a negative match of an IPset for src,src +// Included flag captures the negative or positive match +// MatchType captures match flags +type SetInfo struct { + IPSet *ipsets.IPSet + Included bool + MatchType string // match type can be “src”, “src,dst” or “dst,dst” etc +} + +type Ports struct { + Port int64 + EndPort int64 +} + +type Verdict string + +type Direction string + +type Protocol string + +const ( + // Ingress when packet is entering a container + Ingress Direction = "IN" + // Egress when packet is leaving a container + Egress Direction = "OUT" + // Both applies to both directions + Both Direction = "BOTH" + + // Allowed is accept in linux + Allowed Verdict = "ALLOW" + // Dropped is denying a flow + Dropped Verdict = "DROP" + + // TCP Protocol + TCP Protocol = "tcp" + // UDP Protocol + UDP Protocol = "udp" + // SCTP Protocol + SCTP Protocol = "sctp" + // ICMP Protocol + ICMP Protocol = "icmp" + // AnyProtocol can be used for all other protocols + AnyProtocol Protocol = "any" +) diff --git a/npm/pkg/dataplane/policies/policy_windows.go b/npm/pkg/dataplane/policies/policy_windows.go new file mode 100644 index 0000000000..5b6d8f6d04 --- /dev/null +++ b/npm/pkg/dataplane/policies/policy_windows.go @@ -0,0 +1,47 @@ +package policies + +import ( + "fmt" + + "github.com/Microsoft/hcsshim/hcn" +) + +var protocolNumMap = map[Protocol]string{ + TCP: "6", + UDP: "17", + ICMP: "1", + SCTP: "132", + // HNS thinks 256 as ANY protocol + AnyProtocol: "256", +} + +func convertToAclSettings(acl ACLPolicy) (hcn.AclPolicySetting, error) { + policySettings := hcn.AclPolicySetting{} + for _, setInfo := range acl.SrcList { + if !setInfo.Included { + return policySettings, fmt.Errorf("Windows Dataplane does not support negative matches. ACL: %+v", acl) + } + } + + return policySettings, nil +} + +func getHCNDirection(direction Direction) hcn.DirectionType { + switch direction { + case Ingress: + return hcn.DirectionTypeIn + case Egress: + return hcn.DirectionTypeOut + } + return "" +} + +func getHCNAction(verdict Verdict) hcn.ActionType { + switch verdict { + case Allowed: + return hcn.ActionTypeAllow + case Dropped: + return hcn.ActionTypeBlock + } + return "" +} diff --git a/npm/pkg/dataplane/policies/policymanager.go b/npm/pkg/dataplane/policies/policymanager.go new file mode 100644 index 0000000000..c228af8a6d --- /dev/null +++ b/npm/pkg/dataplane/policies/policymanager.go @@ -0,0 +1,59 @@ +package policies + +type PolicyMap struct { + cache map[string]*NPMNetworkPolicy +} + +type PolicyManager struct { + policyMap *PolicyMap +} + +func NewPolicyManager() PolicyManager { + return PolicyManager{ + policyMap: &PolicyMap{ + cache: make(map[string]*NPMNetworkPolicy), + }, + } +} + +func (pMgr *PolicyManager) GetPolicy(name string) (*NPMNetworkPolicy, error) { + if policy, ok := pMgr.policyMap.cache[name]; ok { + return policy, nil + } + + return nil, nil +} + +func (pMgr *PolicyManager) AddPolicy(policy *NPMNetworkPolicy) error { + // Call actual dataplane function to apply changes + err := pMgr.addPolicy(policy) + if err != nil { + return err + } + + pMgr.policyMap.cache[policy.Name] = policy + return nil +} + +func (pMgr *PolicyManager) RemovePolicy(name string) error { + // Call actual dataplane function to apply changes + err := pMgr.removePolicy(name) + if err != nil { + return err + } + + delete(pMgr.policyMap.cache, name) + + return nil +} + +func (pMgr *PolicyManager) UpdatePolicy(policy *NPMNetworkPolicy) error { + // check and update + // Call actual dataplane function to apply changes + err := pMgr.updatePolicy(policy) + if err != nil { + return err + } + + return nil +} diff --git a/npm/pkg/dataplane/policies/policymanager_linux.go b/npm/pkg/dataplane/policies/policymanager_linux.go new file mode 100644 index 0000000000..ac1ef7b28d --- /dev/null +++ b/npm/pkg/dataplane/policies/policymanager_linux.go @@ -0,0 +1,13 @@ +package policies + +func (pMgr *PolicyManager) addPolicy(policy *NPMNetworkPolicy) error { + return nil +} + +func (pMgr *PolicyManager) removePolicy(name string) error { + return nil +} + +func (pMgr *PolicyManager) updatePolicy(policy *NPMNetworkPolicy) error { + return nil +} diff --git a/npm/pkg/dataplane/policies/policymanager_test.go b/npm/pkg/dataplane/policies/policymanager_test.go new file mode 100644 index 0000000000..77f7824225 --- /dev/null +++ b/npm/pkg/dataplane/policies/policymanager_test.go @@ -0,0 +1,39 @@ +package policies + +import "testing" + +func TestAddPolicy(t *testing.T) { + pMgr := NewPolicyManager() + + netpol := NPMNetworkPolicy{} + + err := pMgr.AddPolicy(&netpol) + if err != nil { + t.Errorf("AddPolicy() returned error %s", err.Error()) + } +} + +func TestRemovePolicy(t *testing.T) { + pMgr := NewPolicyManager() + + err := pMgr.RemovePolicy("test") + if err != nil { + t.Errorf("RemovePolicy() returned error %s", err.Error()) + } +} + +func TestUpdatePolicy(t *testing.T) { + pMgr := NewPolicyManager() + + netpol := NPMNetworkPolicy{} + + err := pMgr.AddPolicy(&netpol) + if err != nil { + t.Errorf("UpdatePolicy() returned error %s", err.Error()) + } + + err = pMgr.UpdatePolicy(&netpol) + if err != nil { + t.Errorf("UpdatePolicy() returned error %s", err.Error()) + } +} diff --git a/npm/pkg/dataplane/policies/policymanager_windows.go b/npm/pkg/dataplane/policies/policymanager_windows.go new file mode 100644 index 0000000000..ac1ef7b28d --- /dev/null +++ b/npm/pkg/dataplane/policies/policymanager_windows.go @@ -0,0 +1,13 @@ +package policies + +func (pMgr *PolicyManager) addPolicy(policy *NPMNetworkPolicy) error { + return nil +} + +func (pMgr *PolicyManager) removePolicy(name string) error { + return nil +} + +func (pMgr *PolicyManager) updatePolicy(policy *NPMNetworkPolicy) error { + return nil +} diff --git a/npm/util/const.go b/npm/util/const.go index 5439b02c7c..c177f1c2cd 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -145,6 +145,8 @@ const ( NamespacePrefix string = "ns-" NegationPrefix string = "not-" + + SetPolicyDelimiter string = "," ) // NPM telemetry constants. diff --git a/npm/util/util.go b/npm/util/util.go index f2b1421acb..ea56410e27 100644 --- a/npm/util/util.go +++ b/npm/util/util.go @@ -330,3 +330,7 @@ func CompareSlices(list1, list2 []string) bool { } return true } + +func SliceToString(list []string) string { + return strings.Join(list, SetPolicyDelimiter) +}