From b95cf6e8171de328292eaa11156dbb5c9b509ca3 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 30 Mar 2022 17:30:49 -0700 Subject: [PATCH 01/17] wip --- npm/pkg/dataplane/ipsets/dirtycache.go | 28 + npm/pkg/dataplane/ipsets/dirtycache_linux.go | 122 + .../dataplane/ipsets/dirtycache_windows.go | 77 + npm/pkg/dataplane/ipsets/ipset.go | 6 +- npm/pkg/dataplane/ipsets/ipsetmanager.go | 61 +- .../dataplane/ipsets/ipsetmanager_linux.go | 382 +-- .../ipsets/ipsetmanager_linux_test.go | 2896 ++++++++--------- npm/pkg/dataplane/ipsets/ipsetmanager_test.go | 13 +- .../dataplane/ipsets/ipsetmanager_windows.go | 9 +- npm/pkg/dataplane/ipsets/testutils_linux.go | 8 +- 10 files changed, 1911 insertions(+), 1691 deletions(-) create mode 100644 npm/pkg/dataplane/ipsets/dirtycache.go create mode 100644 npm/pkg/dataplane/ipsets/dirtycache_linux.go create mode 100644 npm/pkg/dataplane/ipsets/dirtycache_windows.go diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go new file mode 100644 index 0000000000..b034fa67b1 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -0,0 +1,28 @@ +package ipsets + +// dirtyCacheMaintainer will maintain the dirty cache +type dirtyCacheMaintainer interface { + // reset empties dirty cache + reset() + // create will mark the new set to be created. + create(newSet *IPSet) + // update will mark the set to be updated and may note the original members of the set. + update(originalSet *IPSet) + // delete will mark the set to be deleted in the cache + delete(originalSet *IPSet) + // getSetsToAddOrUpdate returns the list of set names to be added or updated + getSetsToAddOrUpdate() []string + // getSetsToDelete returns the list of set names to be deleted + getSetsToDelete() []string + // numSetsToAddOrUpdate returns the number of sets to be added or updated + numSetsToAddOrUpdate() int + // numSetsToDelete returns the number of sets to be deleted + numSetsToDelete() int + // isSetToAddOrUpdate returns true if the set is dirty and should be added or updated + isSetToAddOrUpdate(setName string) bool + // isSetToDelete returns true if the set is dirty and should be deleted + isSetToDelete(setName string) bool + // getOriginalMembers returns the original members of the set before it was dirty + // members are either IPs, CIDRs, IP-Port pairs, or set names if the parent is a list + getOriginalMembers(setName string) map[string]struct{} +} diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go new file mode 100644 index 0000000000..f663412db3 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -0,0 +1,122 @@ +package ipsets + +type memberAwareDirtyCache struct { + toAddOrUpdateCache map[string]*dirtyInfo + toDeleteCache map[string]*dirtyInfo +} + +type dirtyInfo struct { + setType SetType + members map[string]struct{} +} + +func newDirtyCache() dirtyCacheMaintainer { + return &memberAwareDirtyCache{ + toAddOrUpdateCache: make(map[string]*dirtyInfo), + toDeleteCache: make(map[string]*dirtyInfo), + } +} + +func (dc *memberAwareDirtyCache) reset() { + dc.toAddOrUpdateCache = make(map[string]*dirtyInfo) + dc.toDeleteCache = make(map[string]*dirtyInfo) +} + +func (dc *memberAwareDirtyCache) create(newSet *IPSet) { + setName := newSet.Name + if _, ok := dc.toAddOrUpdateCache[setName]; ok { + // NOTE: could throw error if setType is different + return + } + info, ok := dc.toDeleteCache[setName] + if !ok { + info = &dirtyInfo{ + setType: newSet.Type, + members: make(map[string]struct{}), + } + } + dc.toAddOrUpdateCache[setName] = info + delete(dc.toDeleteCache, setName) +} + +func (dc *memberAwareDirtyCache) update(originalSet *IPSet) { + putIntoAndRemoveFromOther(originalSet, dc.toAddOrUpdateCache, dc.toDeleteCache) +} + +func (dc *memberAwareDirtyCache) delete(originalSet *IPSet) { + putIntoAndRemoveFromOther(originalSet, dc.toDeleteCache, dc.toAddOrUpdateCache) +} + +func putIntoAndRemoveFromOther(originalSet *IPSet, intoCache, fromCache map[string]*dirtyInfo) { + setName := originalSet.Name + if _, ok := intoCache[setName]; ok { + // NOTE: could throw error if setType is different + return + } + info, ok := fromCache[setName] + if !ok { + setType := originalSet.Type + members := make(map[string]struct{}) + if setType.getSetKind() == HashSet { + for member := range originalSet.IPPodKey { + members[member] = struct{}{} + } + } else { + for memberName := range originalSet.MemberIPSets { + members[memberName] = struct{}{} + } + } + info = &dirtyInfo{ + setType: setType, + members: members, + } + } + intoCache[setName] = info + delete(fromCache, setName) +} + +func (dc *memberAwareDirtyCache) getSetsToAddOrUpdate() []string { + setsToAddOrUpdate := make([]string, 0, len(dc.toAddOrUpdateCache)) + for setName := range dc.toAddOrUpdateCache { + setsToAddOrUpdate = append(setsToAddOrUpdate, setName) + } + return setsToAddOrUpdate +} + +func (dc *memberAwareDirtyCache) getSetsToDelete() []string { + setsToDelete := make([]string, 0, len(dc.toDeleteCache)) + for setName := range dc.toDeleteCache { + setsToDelete = append(setsToDelete, setName) + } + return setsToDelete +} + +func (dc *memberAwareDirtyCache) numSetsToAddOrUpdate() int { + return len(dc.toAddOrUpdateCache) +} + +func (dc *memberAwareDirtyCache) numSetsToDelete() int { + return len(dc.toDeleteCache) +} + +func (dc *memberAwareDirtyCache) isSetToAddOrUpdate(setName string) bool { + _, ok := dc.toAddOrUpdateCache[setName] + return ok +} + +func (dc *memberAwareDirtyCache) isSetToDelete(setName string) bool { + _, ok := dc.toDeleteCache[setName] + return ok +} + +func (dc *memberAwareDirtyCache) getOriginalMembers(setName string) map[string]struct{} { + info, ok := dc.toAddOrUpdateCache[setName] + if !ok { + return nil + } + members := make(map[string]struct{}, len(info.members)) + for member := range info.members { + members[member] = struct{}{} + } + return members +} diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go new file mode 100644 index 0000000000..24f08f7caf --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -0,0 +1,77 @@ +package ipsets + +type nameOnlyDirtyCache struct { + toAddOrUpdateCache map[string]struct{} + toDeleteCache map[string]struct{} +} + +func newDirtyCache() dirtyCacheMaintainer { + return &nameOnlyDirtyCache{ + toAddOrUpdateCache: make(map[string]struct{}), + toDeleteCache: make(map[string]struct{}), + } +} + +func (dc *nameOnlyDirtyCache) reset() { + dc.toAddOrUpdateCache = make(map[string]struct{}) + dc.toDeleteCache = make(map[string]struct{}) +} + +func (dc *nameOnlyDirtyCache) create(newSet *IPSet) { + putIntoAndRemoveFromOther(newSet, dc.toAddOrUpdateCache, dc.toDeleteCache) +} + +func (dc *nameOnlyDirtyCache) update(originalSet *IPSet) { + putIntoAndRemoveFromOther(originalSet, dc.toAddOrUpdateCache, dc.toDeleteCache) +} + +func (dc *nameOnlyDirtyCache) delete(originalSet *IPSet) { + putIntoAndRemoveFromOther(originalSet, dc.toDeleteCache, dc.toAddOrUpdateCache) +} + +func putIntoAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struct{}) { + if _, ok := intoCache[set.Name]; ok { + // NOTE: could throw error if setType is different + return + } + intoCache[set.Name] = struct{}{} + delete(fromCache, set.Name) +} + +func (dc *nameOnlyDirtyCache) getSetsToAddOrUpdate() []string { + result := make([]string, 0, len(dc.toAddOrUpdateCache)) + for setName := range dc.toAddOrUpdateCache { + result = append(result, setName) + } + return result +} + +func (dc *nameOnlyDirtyCache) getSetsToDelete() []string { + result := make([]string, 0, len(dc.toDeleteCache)) + for setName := range dc.toDeleteCache { + result = append(result, setName) + } + return result +} + +func (dc *nameOnlyDirtyCache) numSetsToAddOrUpdate() int { + return len(dc.toAddOrUpdateCache) +} + +func (dc *nameOnlyDirtyCache) numSetsToDelete() int { + return len(dc.toDeleteCache) +} + +func (dc *nameOnlyDirtyCache) isSetToAddOrUpdate(setName string) bool { + _, ok := dc.toAddOrUpdateCache[setName] + return ok +} + +func (dc *nameOnlyDirtyCache) isSetToDelete(setName string) bool { + _, ok := dc.toDeleteCache[setName] + return ok +} + +func (dc *nameOnlyDirtyCache) getOriginalMembers(_ string) map[string]struct{} { + return nil +} diff --git a/npm/pkg/dataplane/ipsets/ipset.go b/npm/pkg/dataplane/ipsets/ipset.go index 18b9c2b613..8f6f64ac49 100644 --- a/npm/pkg/dataplane/ipsets/ipset.go +++ b/npm/pkg/dataplane/ipsets/ipset.go @@ -71,7 +71,11 @@ func (setMetadata *IPSetMetadata) GetPrefixName() string { } func (setMetadata *IPSetMetadata) GetSetKind() SetKind { - switch setMetadata.Type { + return setMetadata.Type.getSetKind() +} + +func (setType SetType) getSetKind() SetKind { + switch setType { case CIDRBlocks: return HashSet case Namespace: diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index 29bab5b12e..b8dafb4505 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -34,14 +34,10 @@ const ( ) type IPSetManager struct { - iMgrCfg *IPSetManagerCfg - setMap map[string]*IPSet - // Map with Key as IPSet name to to emulate set - // and value as struct{} for minimal memory consumption. - toAddOrUpdateCache map[string]struct{} - // IPSets referred to in this cache may be in the setMap, but must be deleted from the kernel - toDeleteCache map[string]struct{} - ioShim *common.IOShim + iMgrCfg *IPSetManagerCfg + setMap map[string]*IPSet + dirtyCache dirtyCacheMaintainer + ioShim *common.IOShim sync.Mutex } @@ -52,11 +48,10 @@ type IPSetManagerCfg struct { func NewIPSetManager(iMgrCfg *IPSetManagerCfg, ioShim *common.IOShim) *IPSetManager { return &IPSetManager{ - iMgrCfg: iMgrCfg, - setMap: make(map[string]*IPSet), - toAddOrUpdateCache: make(map[string]struct{}), - toDeleteCache: make(map[string]struct{}), - ioShim: ioShim, + iMgrCfg: iMgrCfg, + setMap: make(map[string]*IPSet), + dirtyCache: newDirtyCache(), + ioShim: ioShim, } } @@ -79,7 +74,8 @@ func (iMgr *IPSetManager) Reconcile() { } numRemovedSets := originalNumSets - len(iMgr.setMap) if numRemovedSets > 0 { - klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache to: %+v", numRemovedSets, iMgr.toDeleteCache) + // FIXME uncomment this + // klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache to: %+v", numRemovedSets, iMgr.toDeleteCache) } } @@ -117,7 +113,7 @@ func (iMgr *IPSetManager) createAndGetIPSet(setMetadata *IPSetMetadata) *IPSet { iMgr.setMap[prefixedName] = set metrics.IncNumIPSets() if iMgr.iMgrCfg.IPSetMode == ApplyAllIPSets { - iMgr.modifyCacheForKernelCreation(prefixedName) + iMgr.modifyCacheForKernelCreation(set) } return set } @@ -160,7 +156,7 @@ func (iMgr *IPSetManager) AddReference(setMetadata *IPSetMetadata, referenceName if !wasInKernel { // the set should be in the kernel, so add it to the kernel if it wasn't beforehand // this branch can only be taken for ApplyOnNeed mode - iMgr.modifyCacheForKernelCreation(set.Name) + iMgr.modifyCacheForKernelCreation(set) // for ApplyAllIPSets mode, the set either: // a) existed already and doesn't need to be added to toAddOrUpdateCache @@ -195,7 +191,7 @@ func (iMgr *IPSetManager) DeleteReference(setName, referenceName string, referen if wasInKernel && !iMgr.shouldBeInKernel(set) { // remove from kernel if it was in the kernel before and shouldn't be now // this branch can only be taken for ApplyOnNeed mode - iMgr.modifyCacheForKernelRemoval(set.Name) + iMgr.modifyCacheForKernelRemoval(set) // for ApplyAllIPSets mode, we don't want to make the set dirty @@ -424,12 +420,13 @@ func (iMgr *IPSetManager) ApplyIPSets() error { iMgr.Lock() defer iMgr.Unlock() - if len(iMgr.toAddOrUpdateCache) == 0 && len(iMgr.toDeleteCache) == 0 { + if iMgr.dirtyCache.numSetsToAddOrUpdate() == 0 && iMgr.dirtyCache.numSetsToDelete() == 0 { klog.Info("[IPSetManager] No IPSets to apply") return nil } - klog.Infof("[IPSetManager] toAddUpdateCache: %+v \ntoDeleteCache: %+v", iMgr.toAddOrUpdateCache, iMgr.toDeleteCache) + // FIXME uncomment this + // klog.Infof("[IPSetManager] toAddUpdateCache: %+v \ntoDeleteCache: %+v", iMgr.toAddOrUpdateCache, iMgr.toDeleteCache) iMgr.sanitizeDirtyCache() // Call the appropriate apply ipsets @@ -480,14 +477,13 @@ func (iMgr *IPSetManager) modifyCacheForCacheDeletion(set *IPSet, deleteOption u if iMgr.iMgrCfg.IPSetMode == ApplyAllIPSets { // NOTE: in ApplyAllIPSets mode, if this ipset has never been created in the kernel, // it would be added to the deleteCache, and then the OS would fail to delete it - iMgr.modifyCacheForKernelRemoval(set.Name) + iMgr.modifyCacheForKernelRemoval(set) } // if mode is ApplyOnNeed, the set will not be in the kernel (or will be in the delete cache already) since there are no references } -func (iMgr *IPSetManager) modifyCacheForKernelCreation(setName string) { - iMgr.toAddOrUpdateCache[setName] = struct{}{} - delete(iMgr.toDeleteCache, setName) +func (iMgr *IPSetManager) modifyCacheForKernelCreation(set *IPSet) { + iMgr.dirtyCache.create(set) /* TODO kernel-based prometheus metrics @@ -501,7 +497,7 @@ func (iMgr *IPSetManager) incKernelReferCountAndModifyCache(member *IPSet) { wasInKernel := iMgr.shouldBeInKernel(member) member.incKernelReferCount() if !wasInKernel { - iMgr.modifyCacheForKernelCreation(member.Name) + iMgr.modifyCacheForKernelCreation(member) } } @@ -509,9 +505,8 @@ func (iMgr *IPSetManager) shouldBeInKernel(set *IPSet) bool { return set.shouldBeInKernel() || iMgr.iMgrCfg.IPSetMode == ApplyAllIPSets } -func (iMgr *IPSetManager) modifyCacheForKernelRemoval(setName string) { - iMgr.toDeleteCache[setName] = struct{}{} - delete(iMgr.toAddOrUpdateCache, setName) +func (iMgr *IPSetManager) modifyCacheForKernelRemoval(set *IPSet) { + iMgr.dirtyCache.delete(set) /* TODO kernel-based prometheus metrics @@ -524,13 +519,13 @@ func (iMgr *IPSetManager) modifyCacheForKernelRemoval(setName string) { func (iMgr *IPSetManager) decKernelReferCountAndModifyCache(member *IPSet) { member.decKernelReferCount() if !iMgr.shouldBeInKernel(member) { - iMgr.modifyCacheForKernelRemoval(member.Name) + iMgr.modifyCacheForKernelRemoval(member) } } func (iMgr *IPSetManager) modifyCacheForKernelMemberUpdate(set *IPSet) { if iMgr.shouldBeInKernel(set) { - iMgr.toAddOrUpdateCache[set.Name] = struct{}{} + iMgr.dirtyCache.create(set) } } @@ -538,9 +533,8 @@ func (iMgr *IPSetManager) modifyCacheForKernelMemberUpdate(set *IPSet) { // if so will not delete it func (iMgr *IPSetManager) sanitizeDirtyCache() { anyProblems := false - for setName := range iMgr.toDeleteCache { - _, ok := iMgr.toAddOrUpdateCache[setName] - if ok { + for _, setName := range iMgr.dirtyCache.getSetsToDelete() { + if iMgr.dirtyCache.isSetToAddOrUpdate(setName) { klog.Errorf("[IPSetManager] Unexpected state in dirty cache %s set is part of both update and delete caches", setName) anyProblems = true } @@ -551,8 +545,7 @@ func (iMgr *IPSetManager) sanitizeDirtyCache() { } func (iMgr *IPSetManager) clearDirtyCache() { - iMgr.toAddOrUpdateCache = make(map[string]struct{}) - iMgr.toDeleteCache = make(map[string]struct{}) + iMgr.dirtyCache.reset() } // validateIPSetMemberIP helps valid if a member added to an HashSet has valid IP or CIDR diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 7e5aec6f41..93c28601ec 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -2,7 +2,6 @@ package ipsets import ( "fmt" - "strings" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/parse" "github.com/Azure/azure-container-networking/npm/util" @@ -180,6 +179,8 @@ func (iMgr *IPSetManager) fileCreatorForReset(ipsetListOutput []byte) (*ioutil.F } /* +FIXME update documentation + overall error handling for ipset restore file. ipset restore will apply all lines to the kernel before a failure, so when recovering from a line failure, we must skip the lines that were already applied. below, "set" refers to either hashset or list, except in the sections for adding to (hash)set and adding to list @@ -248,13 +249,14 @@ example where every set in add/update cache should have ip 1.2.3.4 and 2.3.4.5: */ func (iMgr *IPSetManager) applyIPSets() error { var saveFile []byte - var saveError error - if len(iMgr.toAddOrUpdateCache) > 0 { - saveFile, saveError = iMgr.ipsetSave() - if saveError != nil { - return npmerrors.SimpleErrorWrapper("ipset save failed when applying ipsets", saveError) - } - } + // FIXME delete + // var saveError error + // if iMgr.dirtyCache.numSetsToAddOrUpdate() > 0 { + // saveFile, saveError = iMgr.ipsetSave() + // if saveError != nil { + // return npmerrors.SimpleErrorWrapper("ipset save failed when applying ipsets", saveError) + // } + // } creator := iMgr.fileCreatorForApply(maxTryCount, saveFile) restoreError := creator.RunCommandWithFile(ipsetCommand, ipsetRestoreFlag) if restoreError != nil { @@ -276,37 +278,52 @@ func (iMgr *IPSetManager) ipsetSave() ([]byte, error) { return saveFile, nil } +// FIXME remove saveFile param func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) *ioutil.FileCreator { creator := ioutil.NewFileCreator(iMgr.ioShim, maxTryCount, ipsetRestoreLineFailurePattern) // TODO make the line failure pattern into a definition constant eventually // 1. create all sets first so we don't try to add a member set to a list if it hasn't been created yet - for prefixedName := range iMgr.toAddOrUpdateCache { + setsToAddOrUpdate := iMgr.dirtyCache.getSetsToAddOrUpdate() + for _, prefixedName := range setsToAddOrUpdate { set := iMgr.setMap[prefixedName] iMgr.createSetForApply(creator, set) // NOTE: currently no logic to handle this scenario: // if a set in the toAddOrUpdateCache is in the kernel with the wrong type, then we'll try to create it, which will fail in the first restore call, but then be skipped in a retry } - // 2. for dirty sets already in the kernel, update members (add members not in the kernel, and delete undesired members in the kernel) - iMgr.updateDirtyKernelSets(saveFile, creator) - - // 3. for the remaining dirty sets, add their members to the kernel - for prefixedName := range iMgr.toAddOrUpdateCache { - set := iMgr.setMap[prefixedName] + // 2. add/delete members from dirty sets to add or update + for _, prefixedName := range setsToAddOrUpdate { sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) + originalMembers := iMgr.dirtyCache.getOriginalMembers(prefixedName) + set := iMgr.setMap[prefixedName] if set.Kind == HashSet { for ip := range set.IPPodKey { - iMgr.addMemberForApply(creator, set, sectionID, ip) + if _, ok := originalMembers[ip]; ok { + // remove from members so we don't try to delete it later + delete(originalMembers, ip) + } else { + // add the member since it didn't exist before + iMgr.addMemberForApply(creator, set, sectionID, ip) + } } } else { - for _, member := range set.MemberIPSets { - iMgr.addMemberForApply(creator, set, sectionID, member.HashedName) + for setName := range set.MemberIPSets { + if _, ok := originalMembers[setName]; ok { + // remove from members so we don't try to delete it later + delete(originalMembers, setName) + } else { + // add the member since it didn't exist before + iMgr.addMemberForApply(creator, set, sectionID, setName) + } } } + for member := range originalMembers { + iMgr.deleteMemberForApply(creator, set, sectionID, member) + } } /* - 4. flush and destroy sets in the original delete cache + 3. flush and destroy sets in the original delete cache We must perform this step after member deletions because of the following scenario: Suppose we want to destroy set A, which is referenced by list L. For set A to be in the toDeleteCache, @@ -314,176 +331,16 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) but until then, set A is in use by a kernel component and can't be destroyed. */ // flush all sets first in case a set we're destroying is referenced by a list we're destroying - for prefixedName := range iMgr.toDeleteCache { + setsToDelete := iMgr.dirtyCache.getSetsToDelete() + for _, prefixedName := range setsToDelete { iMgr.flushSetForApply(creator, prefixedName) } - for prefixedName := range iMgr.toDeleteCache { + for _, prefixedName := range setsToDelete { iMgr.destroySetForApply(creator, prefixedName) } return creator } -// updates the creator (adds/deletes members) for dirty sets already in the kernel -// updates the toAddOrUpdateCache: after calling this function, the cache will only consist of sets to create -// error handling principal: -// - if contract with ipset save (or grep) is breaking, salvage what we can, take a snapshot (TODO), and log the failure -// - have a background process for sending/removing snapshots intermittently -func (iMgr *IPSetManager) updateDirtyKernelSets(saveFile []byte, creator *ioutil.FileCreator) { - // map hashed names to prefixed names - toAddOrUpdateHashedNames := make(map[string]string) - for prefixedName := range iMgr.toAddOrUpdateCache { - hashedName := iMgr.setMap[prefixedName].HashedName - toAddOrUpdateHashedNames[hashedName] = prefixedName - } - - // in each iteration, read a create line and any ensuing add lines - readIndex := 0 - var line []byte - if readIndex < len(saveFile) { - line, readIndex = parse.Line(readIndex, saveFile) - if !hasPrefix(line, createStringWithSpace) { - klog.Errorf("expected a create line in ipset save file, but got the following line: %s", string(line)) - // TODO send error snapshot - line, readIndex = nextCreateLine(readIndex, saveFile) - } - } - for readIndex < len(saveFile) { - // 1. get the hashed name - lineAfterCreate := string(line[len(createStringWithSpace):]) - spaceSplitLineAfterCreate := strings.Split(lineAfterCreate, space) - hashedName := spaceSplitLineAfterCreate[0] - - // 2. continue to the next create line if the set isn't in the toAddOrUpdateCache - prefixedName, shouldModify := toAddOrUpdateHashedNames[hashedName] - if !shouldModify { - line, readIndex = nextCreateLine(readIndex, saveFile) - continue - } - - // 3. update the set from the kernel - set := iMgr.setMap[prefixedName] - // remove from the dirty cache so we don't add it later - delete(iMgr.toAddOrUpdateCache, prefixedName) - // mark the set as in the kernel - delete(toAddOrUpdateHashedNames, hashedName) - - // 3.1 check for consistent type - restOfLine := spaceSplitLineAfterCreate[1:] - if haveTypeProblem(set, restOfLine) { - // error logging happens in the helper function - // TODO send error snapshot - line, readIndex = nextCreateLine(readIndex, saveFile) - continue - } - - // 3.2 get desired members from cache - var membersToAdd map[string]struct{} - if set.Kind == HashSet { - membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) - for ip := range set.IPPodKey { - membersToAdd[ip] = struct{}{} - } - } else { - membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) - for _, member := range set.MemberIPSets { - membersToAdd[member.HashedName] = struct{}{} - } - } - - // 3.4 determine which members to add/delete - membersToDelete := make(map[string]struct{}) - for readIndex < len(saveFile) { - line, readIndex = parse.Line(readIndex, saveFile) - if hasPrefix(line, createStringWithSpace) { - break - } - if !hasPrefix(line, addStringWithSpace) { - klog.Errorf("expected an add line, but got the following line: %s", string(line)) - // TODO send error snapshot - line, readIndex = nextCreateLine(readIndex, saveFile) - break - } - lineAfterAdd := string(line[len(addStringWithSpace):]) - spaceSplitLineAfterAdd := strings.Split(lineAfterAdd, space) - parent := spaceSplitLineAfterAdd[0] - if len(spaceSplitLineAfterAdd) != 2 || parent != hashedName { - klog.Errorf("expected an add line for set %s in ipset save file, but got the following line: %s", hashedName, string(line)) - // TODO send error snapshot - line, readIndex = nextCreateLine(readIndex, saveFile) - break - } - member := spaceSplitLineAfterAdd[1] - - _, shouldKeep := membersToAdd[member] - if shouldKeep { - // member already in the kernel, so don't add it later - delete(membersToAdd, member) - } else { - // member should be deleted from the kernel - membersToDelete[member] = struct{}{} - } - } - - // 3.5 delete undesired members from restore file - sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) - for member := range membersToDelete { - iMgr.deleteMemberForApply(creator, set, sectionID, member) - } - // 3.5 add new members to restore file - for member := range membersToAdd { - iMgr.addMemberForApply(creator, set, sectionID, member) - } - } -} - -func nextCreateLine(originalReadIndex int, saveFile []byte) (createLine []byte, nextReadIndex int) { - nextReadIndex = originalReadIndex - for nextReadIndex < len(saveFile) { - createLine, nextReadIndex = parse.Line(nextReadIndex, saveFile) - if hasPrefix(createLine, createStringWithSpace) { - return - } - } - return -} - -func haveTypeProblem(set *IPSet, restOfSpaceSplitCreateLine []string) bool { - // TODO check type based on maxelem for hash sets? CIDR blocks have a different maxelem - if len(restOfSpaceSplitCreateLine) == 0 { - klog.Error("expected a type specification for the create line but received nothing after the set name") - return true - } - typeString := restOfSpaceSplitCreateLine[0] - switch typeString { - case ipsetSetListString: - if set.Kind != ListSet { - lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) - klog.Errorf("expected to find a ListSet but have the line: %s", lineString) - return true - } - case ipsetNetHashString: - if set.Kind != HashSet || set.Type == NamedPorts { - lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) - klog.Errorf("expected to find a non-NamedPorts HashSet but have the following line: %s", lineString) - return true - } - case ipsetIPPortHashString: - if set.Type != NamedPorts { - lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) - klog.Errorf("expected to find a NamedPorts set but have the following line: %s", lineString) - return true - } - default: - klog.Errorf("unknown type string [%s] in line: %s", typeString, strings.Join(restOfSpaceSplitCreateLine, " ")) - return true - } - return false -} - -func hasPrefix(line []byte, prefix string) bool { - return len(line) >= len(prefix) && string(line[:len(prefix)]) == prefix -} - func (iMgr *IPSetManager) flushSetForApply(creator *ioutil.FileCreator, prefixedName string) { errorHandlers := []*ioutil.LineErrorHandler{ { @@ -617,3 +474,164 @@ func (iMgr *IPSetManager) addMemberForApply(creator *ioutil.FileCreator, set *IP func sectionID(prefix, prefixedName string) string { return fmt.Sprintf("%s-%s", prefix, prefixedName) } + +// updates the creator (adds/deletes members) for dirty sets already in the kernel +// updates the toAddOrUpdateCache: after calling this function, the cache will only consist of sets to create +// error handling principal: +// - if contract with ipset save (or grep) is breaking, salvage what we can, take a snapshot (TODO), and log the failure +// - have a background process for sending/removing snapshots intermittently +// func (iMgr *IPSetManager) updateDirtyKernelSets(saveFile []byte, creator *ioutil.FileCreator) { +// // map hashed names to prefixed names +// toAddOrUpdateHashedNames := make(map[string]string) +// for prefixedName := range iMgr.toAddOrUpdateCache { +// hashedName := iMgr.setMap[prefixedName].HashedName +// toAddOrUpdateHashedNames[hashedName] = prefixedName +// } + +// // in each iteration, read a create line and any ensuing add lines +// readIndex := 0 +// var line []byte +// if readIndex < len(saveFile) { +// line, readIndex = parse.Line(readIndex, saveFile) +// if !hasPrefix(line, createStringWithSpace) { +// klog.Errorf("expected a create line in ipset save file, but got the following line: %s", string(line)) +// // TODO send error snapshot +// line, readIndex = nextCreateLine(readIndex, saveFile) +// } +// } +// for readIndex < len(saveFile) { +// // 1. get the hashed name +// lineAfterCreate := string(line[len(createStringWithSpace):]) +// spaceSplitLineAfterCreate := strings.Split(lineAfterCreate, space) +// hashedName := spaceSplitLineAfterCreate[0] + +// // 2. continue to the next create line if the set isn't in the toAddOrUpdateCache +// prefixedName, shouldModify := toAddOrUpdateHashedNames[hashedName] +// if !shouldModify { +// line, readIndex = nextCreateLine(readIndex, saveFile) +// continue +// } + +// // 3. update the set from the kernel +// set := iMgr.setMap[prefixedName] +// // remove from the dirty cache so we don't add it later +// delete(iMgr.toAddOrUpdateCache, prefixedName) +// // mark the set as in the kernel +// delete(toAddOrUpdateHashedNames, hashedName) + +// // 3.1 check for consistent type +// restOfLine := spaceSplitLineAfterCreate[1:] +// if haveTypeProblem(set, restOfLine) { +// // error logging happens in the helper function +// // TODO send error snapshot +// line, readIndex = nextCreateLine(readIndex, saveFile) +// continue +// } + +// // 3.2 get desired members from cache +// var membersToAdd map[string]struct{} +// if set.Kind == HashSet { +// membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) +// for ip := range set.IPPodKey { +// membersToAdd[ip] = struct{}{} +// } +// } else { +// membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) +// for _, member := range set.MemberIPSets { +// membersToAdd[member.HashedName] = struct{}{} +// } +// } + +// // 3.4 determine which members to add/delete +// membersToDelete := make(map[string]struct{}) +// for readIndex < len(saveFile) { +// line, readIndex = parse.Line(readIndex, saveFile) +// if hasPrefix(line, createStringWithSpace) { +// break +// } +// if !hasPrefix(line, addStringWithSpace) { +// klog.Errorf("expected an add line, but got the following line: %s", string(line)) +// // TODO send error snapshot +// line, readIndex = nextCreateLine(readIndex, saveFile) +// break +// } +// lineAfterAdd := string(line[len(addStringWithSpace):]) +// spaceSplitLineAfterAdd := strings.Split(lineAfterAdd, space) +// parent := spaceSplitLineAfterAdd[0] +// if len(spaceSplitLineAfterAdd) != 2 || parent != hashedName { +// klog.Errorf("expected an add line for set %s in ipset save file, but got the following line: %s", hashedName, string(line)) +// // TODO send error snapshot +// line, readIndex = nextCreateLine(readIndex, saveFile) +// break +// } +// member := spaceSplitLineAfterAdd[1] + +// _, shouldKeep := membersToAdd[member] +// if shouldKeep { +// // member already in the kernel, so don't add it later +// delete(membersToAdd, member) +// } else { +// // member should be deleted from the kernel +// membersToDelete[member] = struct{}{} +// } +// } + +// // 3.5 delete undesired members from restore file +// sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) +// for member := range membersToDelete { +// iMgr.deleteMemberForApply(creator, set, sectionID, member) +// } +// // 3.5 add new members to restore file +// for member := range membersToAdd { +// iMgr.addMemberForApply(creator, set, sectionID, member) +// } +// } +// } + +// func nextCreateLine(originalReadIndex int, saveFile []byte) (createLine []byte, nextReadIndex int) { +// nextReadIndex = originalReadIndex +// for nextReadIndex < len(saveFile) { +// createLine, nextReadIndex = parse.Line(nextReadIndex, saveFile) +// if hasPrefix(createLine, createStringWithSpace) { +// return +// } +// } +// return +// } + +// func haveTypeProblem(set *IPSet, restOfSpaceSplitCreateLine []string) bool { +// // TODO check type based on maxelem for hash sets? CIDR blocks have a different maxelem +// if len(restOfSpaceSplitCreateLine) == 0 { +// klog.Error("expected a type specification for the create line but received nothing after the set name") +// return true +// } +// typeString := restOfSpaceSplitCreateLine[0] +// switch typeString { +// case ipsetSetListString: +// if set.Kind != ListSet { +// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) +// klog.Errorf("expected to find a ListSet but have the line: %s", lineString) +// return true +// } +// case ipsetNetHashString: +// if set.Kind != HashSet || set.Type == NamedPorts { +// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) +// klog.Errorf("expected to find a non-NamedPorts HashSet but have the following line: %s", lineString) +// return true +// } +// case ipsetIPPortHashString: +// if set.Type != NamedPorts { +// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) +// klog.Errorf("expected to find a NamedPorts set but have the following line: %s", lineString) +// return true +// } +// default: +// klog.Errorf("unknown type string [%s] in line: %s", typeString, strings.Join(restOfSpaceSplitCreateLine, " ")) +// return true +// } +// return false +// } + +// func hasPrefix(line []byte, prefix string) bool { +// return len(line) >= len(prefix) && string(line[:len(prefix)]) == prefix +// } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go index 3ef7f356e3..74ec3e8aee 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go @@ -1,21 +1,5 @@ package ipsets -import ( - "fmt" - "regexp" - "sort" - "strings" - "testing" - - "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/npm/metrics" - "github.com/Azure/azure-container-networking/npm/metrics/promutil" - dptestutils "github.com/Azure/azure-container-networking/npm/pkg/dataplane/testutils" - "github.com/Azure/azure-container-networking/npm/util" - testutils "github.com/Azure/azure-container-networking/test/utils" - "github.com/stretchr/testify/require" -) - const ( saveResult = "create test-list1 list:set size 8\nadd test-list1 test-list2" @@ -34,1442 +18,1444 @@ var resetIPSetsListOutput = []byte(resetIPSetsListOutputString) // TODO test that a reconcile list is updated for all the TestFailure UTs // TODO same exact TestFailure UTs for unknown errors -func TestApplyIPSets(t *testing.T) { - type args struct { - toAddUpdateSets []*IPSetMetadata - toDeleteSets []*IPSetMetadata - commandError bool - } - tests := []struct { - name string - args args - expectedExecCount int - wantErr bool - }{ - { - name: "nothing to update", - args: args{ - toAddUpdateSets: nil, - toDeleteSets: nil, - commandError: false, - }, - expectedExecCount: 0, - wantErr: false, - }, - { - name: "success with just add", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: nil, - commandError: false, - }, - expectedExecCount: 1, - wantErr: false, - }, - { - name: "success with just delete", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: nil, - commandError: false, - }, - expectedExecCount: 1, - wantErr: false, - }, - { - name: "success with add and delete", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, - commandError: false, - }, - expectedExecCount: 1, - wantErr: false, - }, - { - name: "set is in both delete and add/update cache", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: []*IPSetMetadata{namespaceSet}, - commandError: false, - }, - expectedExecCount: 1, - wantErr: false, - }, - { - name: "apply error", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, - commandError: true, - }, - expectedExecCount: 1, - wantErr: true, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - metrics.ReinitializeAll() - calls := GetApplyIPSetsTestCalls(tt.args.toAddUpdateSets, tt.args.toDeleteSets) - if tt.args.commandError { - // add an error to the last call (the ipset-restore call) - // this would potentially cause problems if we used pointers to the TestCmds - require.Greater(t, len(calls), 0) - calls[len(calls)-1].ExitCode = 1 - // then add errors as many times as we retry - for i := 1; i < maxTryCount; i++ { - calls = append(calls, testutils.TestCmd{Cmd: ipsetRestoreStringSlice, ExitCode: 1}) - } - } - ioShim := common.NewMockIOShim(calls) - defer ioShim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) - iMgr.CreateIPSets(tt.args.toAddUpdateSets) - for _, set := range tt.args.toDeleteSets { - iMgr.toDeleteCache[set.GetPrefixName()] = struct{}{} - } - err := iMgr.ApplyIPSets() - - // cache behavior is currently undefined if there's an apply error - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - cache := make([]setMembers, 0) - for _, set := range tt.args.toAddUpdateSets { - cache = append(cache, setMembers{metadata: set, members: nil}) - } - assertExpectedInfo(t, iMgr, &expectedInfo{ - mainCache: cache, - toAddUpdateCache: nil, - toDeleteCache: nil, - setsForKernel: nil, - }) - } - - execCount, err := metrics.GetIPSetExecCount() - promutil.NotifyIfErrors(t, err) - require.Equal(t, tt.expectedExecCount, execCount) - }) - } -} - -func TestNextCreateLine(t *testing.T) { - createLine := "create test-list1 list:set size 8" - addLine := "add test-set1 1.2.3.4" - createLineWithNewline := createLine + "\n" - addLineWithNewline := addLine + "\n" - tests := []struct { - name string - lines []string - expectedReadIndex int - expectedLine []byte - }{ - // parse.Line will omit the newline at the end of the line unless it's the last line - { - name: "empty save file", - lines: []string{}, - expectedReadIndex: 0, - expectedLine: nil, - }, - { - name: "no creates", - lines: []string{addLineWithNewline}, - expectedReadIndex: len(addLineWithNewline), - expectedLine: []byte(addLineWithNewline), - }, - { - name: "start with create", - lines: []string{createLine, addLineWithNewline}, - expectedReadIndex: len(createLineWithNewline), - expectedLine: []byte(createLine), - }, - { - name: "create after adds", - lines: []string{addLine, addLine, createLineWithNewline}, - expectedReadIndex: 2*len(addLine+"\n") + len(createLine+"\n"), - expectedLine: []byte(createLineWithNewline), - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - saveFile := []byte(strings.Join(tt.lines, "\n")) - line, readIndex := nextCreateLine(0, saveFile) - require.Equal(t, tt.expectedReadIndex, readIndex) - require.Equal(t, tt.expectedLine, line) - }) - } - // fmt.Println(string([]byte(addLine + addLine, createLineWithNewline})[:78])) -} - -func TestDestroyNPMIPSetsCreatorSuccess(t *testing.T) { - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) - actualLines := strings.Split(creator.ToString(), "\n") - expectedLines := []string{ - "-F azure-npm-123456", - "-F azure-npm-987654", - "-F azure-npm-777777", - "-X azure-npm-123456", - "-X azure-npm-987654", - "-X azure-npm-777777", - "", - } - dptestutils.AssertEqualLines(t, expectedLines, actualLines) - require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") - wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.False(t, wasModified) - require.NoError(t, err) - require.Equal(t, 0, *destroyFailureCount, "got unexpected failure count") -} - -func TestDestroyNPMIPSetsCreatorErrorHandling(t *testing.T) { - tests := []struct { - name string - call testutils.TestCmd - expectedLines []string - expectedFailureCount int - }{ - { - name: "set doesn't exist on flush", - call: testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 2: The set with the given name does not exist", - ExitCode: 1, - }, - expectedLines: []string{ - "-F azure-npm-777777", - "-X azure-npm-123456", - "-X azure-npm-777777", - "", - }, - expectedFailureCount: 0, - }, - { - name: "some other error on flush", - call: testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 2: for some other error", - ExitCode: 1, - }, - expectedLines: []string{ - "-F azure-npm-777777", - "-X azure-npm-123456", - "-X azure-npm-777777", - "", - }, - expectedFailureCount: 1, - }, - { - name: "set doesn't exist on destroy", - call: testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 5: The set with the given name does not exist", - ExitCode: 1, - }, - expectedLines: []string{ - "-X azure-npm-777777", - "", - }, - expectedFailureCount: 0, - }, - { - name: "some other error on destroy", - call: testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 5: some other error", - ExitCode: 1, - }, - expectedLines: []string{ - "-X azure-npm-777777", - "", - }, - expectedFailureCount: 1, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - calls := []testutils.TestCmd{tt.call} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) - require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") - wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.True(t, wasModified) - require.Error(t, err) - actualLines := strings.Split(creator.ToString(), "\n") - dptestutils.AssertEqualLines(t, tt.expectedLines, actualLines) - require.Equal(t, tt.expectedFailureCount, *destroyFailureCount, "got unexpected failure count") - }) - } -} - -func TestDestroyNPMIPSets(t *testing.T) { - tests := []struct { - name string - calls []testutils.TestCmd - wantErr bool - }{ - { - name: "success with no results from grep", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, ExitCode: 1}, - }, - wantErr: false, - }, - { - name: "successfully delete sets", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - fakeRestoreSuccessCommand, - }, - wantErr: false, - }, - { - name: "grep error", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, HasStartError: true, PipedToCommand: true, ExitCode: 1}, - {Cmd: []string{"grep", "azure-npm-"}}, - }, - wantErr: true, - }, - { - name: "restore error from max tries", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - }, - wantErr: true, - }, - { - name: "successfully restore, but fail to flush/destroy 1 set since the set doesn't exist when flushing", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - { - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 2: The set with the given name does not exist", - ExitCode: 1, - }, - fakeRestoreSuccessCommand, - }, - wantErr: false, - }, - { - name: "successfully restore, but fail to flush/destroy 1 set due to other flush error", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - { - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 2: for some other error", - ExitCode: 1, - }, - fakeRestoreSuccessCommand, - }, - wantErr: false, - }, - { - name: "successfully restore, but fail to destroy 1 set since the set doesn't exist when destroying", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - { - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 5: The set with the given name does not exist", - ExitCode: 1, - }, - fakeRestoreSuccessCommand, - }, - wantErr: false, - }, - { - name: "successfully restore, but fail to destroy 1 set due to other destroy error", - calls: []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, - { - Cmd: ipsetRestoreStringSlice, - Stdout: "Error in line 5: for some other error", - ExitCode: 1, - }, - fakeRestoreSuccessCommand, - }, - wantErr: false, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - ioshim := common.NewMockIOShim(tt.calls) - defer ioshim.VerifyCalls(t, tt.calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - err := iMgr.resetIPSets() - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - -// identical to TestResetIPSets in ipsetmanager_test.go except an error occurs -// makes sure that the cache and metrics are reset despite error -func TestResetIPSetsOnFailure(t *testing.T) { - metrics.ReinitializeAll() - calls := []testutils.TestCmd{ - {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true, HasStartError: true}, - {Cmd: []string{"grep", "azure-npm-"}}, - } - ioShim := common.NewMockIOShim(calls) - defer ioShim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) - - iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet, keyLabelOfPodSet}) - - metrics.IncNumIPSets() - metrics.IncNumIPSets() - metrics.AddEntryToIPSet("test1") - metrics.AddEntryToIPSet("test1") - metrics.AddEntryToIPSet("test2") - - require.NoError(t, iMgr.ResetIPSets()) - - assertExpectedInfo(t, iMgr, &expectedInfo{ - mainCache: nil, - toAddUpdateCache: nil, - toDeleteCache: nil, - setsForKernel: nil, - }) -} - -func TestApplyIPSetsSuccessWithoutSave(t *testing.T) { - // no sets to add/update, so don't call ipset save - calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // delete a set so the file isn't empty (otherwise the creator won't even call the exec command) - iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestNSSet.PrefixName, util.SoftDelete) - err := iMgr.applyIPSets() - require.NoError(t, err) -} - -func TestApplyIPSetsSuccessWithSave(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}}, - {Cmd: ipsetRestoreStringSlice}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // create a set so we run ipset save - iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) - err := iMgr.applyIPSets() - require.NoError(t, err) -} - -func TestApplyIPSetsFailureOnSave(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, HasStartError: true, PipedToCommand: true, ExitCode: 1}, - {Cmd: []string{"grep", "azure-npm-"}}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // create a set so we run ipset save - iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) - err := iMgr.applyIPSets() - require.Error(t, err) -} - -func TestApplyIPSetsFailureOnRestore(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}}, - // fail 3 times because this is our max try count - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // create a set so we run ipset save - iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) - err := iMgr.applyIPSets() - require.Error(t, err) -} - -func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}}, - {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, - {Cmd: ipsetRestoreStringSlice}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // create a set so we run ipset save - iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) - err := iMgr.applyIPSets() - require.NoError(t, err) -} - -func TestIPSetSave(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - output, err := iMgr.ipsetSave() - require.NoError(t, err) - require.Equal(t, saveResult, string(output)) -} - -func TestIPSetSaveNoMatch(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, ExitCode: 1}, - {Cmd: []string{"grep", "azure-npm-"}}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - output, err := iMgr.ipsetSave() - require.NoError(t, err) - require.Nil(t, output) -} - -func TestCreateForAllSetTypes(t *testing.T) { - // without save file - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) - iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) - - creator := iMgr.fileCreatorForApply(len(calls), nil) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - - expectedLines := []string{ - fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), - fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), - fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), - fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), - fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), - "", - } - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err, "ipset restore should be successful") - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestDestroy(t *testing.T) { - // without save file - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // remove some members and destroy some sets - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) - require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), nil) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - - expectedLines := []string{ - fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - fmt.Sprintf("-F %s", TestCIDRSet.HashedName), - fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), - fmt.Sprintf("-X %s", TestCIDRSet.HashedName), - fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), - "", - } - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err, "ipset restore should be successful") - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestUpdateWithIdenticalSaveFile(t *testing.T) { - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), - fmt.Sprintf("add %s 10.0.0.5", TestKeyPodSet.HashedName), - fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), - fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), - fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), - fmt.Sprintf(createListFormat, TestKVNSList.HashedName), - fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), - fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) - iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - - expectedLines := []string{ - fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), - fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), - "", - } - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err, "ipset restore should be successful") - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestUpdateWithRealisticSaveFile(t *testing.T) { - // save file doesn't have some sets we're adding and has some sets that: - // - aren't dirty - // - will be deleted - // - have members which we will delete - // - are missing members, which we will add - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), // should add 10.0.0.1-5 to this set - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member - fmt.Sprintf("add %s 5.6.7.8", TestNSSet.HashedName), // delete this member - fmt.Sprintf("add %s 5.6.7.9", TestNSSet.HashedName), // delete this member - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // dirty but no member changes in the end - fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), // ignore this set since it's not dirty - fmt.Sprintf("add %s 1.2.3.4", TestKVPodSet.HashedName), // ignore this set since it's not dirty - fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), // should add TestKeyPodSet to this set - fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // keep this member - fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), // delete this member - fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), // ignore this set since it's not dirty - fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), // this set will be deleted - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.2", "c")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.3", "d")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.4", "e")) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.5", "f")) - iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestKVNSList.Metadata}) - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "z")) // set not in save file - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) // adding NSSet and KeyPodSet (should be keeping NSSet and deleting NamedportSet) - - expectedLines := []string{ - fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), - fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), - fmt.Sprintf("-A %s 1.2.3.4", TestCIDRSet.HashedName), - fmt.Sprintf("-D %s 5.6.7.8", TestNSSet.HashedName), - fmt.Sprintf("-D %s 5.6.7.9", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.2", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.3", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.4", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.5", TestNSSet.HashedName), - fmt.Sprintf("-D %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), - fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), - fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), - "", - } - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err, "ipset restore should be successful") - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestHaveTypeProblem(t *testing.T) { - type args struct { - metadata *IPSetMetadata - format string - } - tests := []struct { - name string - args args - wantProblem bool - }{ - { - name: "correct type for nethash", - args: args{ - TestNSSet.Metadata, - createNethashFormat, - }, - wantProblem: false, - }, - { - name: "nethash instead of porthash", - args: args{ - TestNamedportSet.Metadata, - createNethashFormat, - }, - wantProblem: true, - }, - { - name: "nethash instead of list", - args: args{ - TestKeyNSList.Metadata, - createNethashFormat, - }, - wantProblem: true, - }, - { - name: "correct type for porthash", - args: args{ - TestNamedportSet.Metadata, - createPorthashFormat, - }, - wantProblem: false, - }, - { - name: "porthash instead of nethash", - args: args{ - TestNSSet.Metadata, - createPorthashFormat, - }, - wantProblem: true, - }, - { - name: "porthash instead of list", - args: args{ - TestKeyNSList.Metadata, - createPorthashFormat, - }, - wantProblem: true, - }, - { - name: "correct type for list", - args: args{ - TestKeyNSList.Metadata, - createListFormat, - }, - wantProblem: false, - }, - { - name: "list instead of nethash", - args: args{ - TestNSSet.Metadata, - createListFormat, - }, - wantProblem: true, - }, - { - name: "list instead of porthash", - args: args{ - TestNamedportSet.Metadata, - createListFormat, - }, - wantProblem: true, - }, - { - name: "unknown type", - args: args{ - TestKeyNSList.Metadata, - "create %s unknown-type", - }, - wantProblem: true, - }, - { - name: "no rest of line", - args: args{ - TestKeyNSList.Metadata, - "create %s", - }, - wantProblem: true, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - set := NewIPSet(tt.args.metadata) - line := fmt.Sprintf(tt.args.format, set.HashedName) - splitLine := strings.Split(line, " ") - restOfLine := splitLine[2:] - if tt.wantProblem { - require.True(t, haveTypeProblem(set, restOfLine)) - } else { - require.False(t, haveTypeProblem(set, restOfLine)) - } - }) - } -} - -func TestUpdateWithBadSaveFile(t *testing.T) { - type args struct { - dirtySet []*IPSetMetadata - saveFileLines []string - } - tests := []struct { - name string - args args - expectedLines []string - }{ - { - name: "no create line", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), - fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), - }, - }, - expectedLines: []string{ - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - "", - }, - }, - { - name: "unexpected verb after create", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), - "wrong-verb ...", - }, - }, - expectedLines: []string{ - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - "", - }, - }, - { - name: "non-NPM set", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - "create test-set1 hash:net family inet hashsize 1024 maxelem 65536", - "add test-set1 1.2.3.4", - }, - }, - expectedLines: []string{ - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - "", - }, - }, - { - name: "ignore set we've already parsed", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include - fmt.Sprintf("add %s 4.4.4.4", TestKeyPodSet.HashedName), // include this add (will DELETE this member) - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // ignore this create and ensuing adds since we already included this set - fmt.Sprintf("add %s 5.5.5.5", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) - }, - }, - expectedLines: []string{ - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-D %s 4.4.4.4", TestKeyPodSet.HashedName), - "", - }, - }, - { - name: "set with wrong type", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - fmt.Sprintf(createPorthashFormat, TestKeyPodSet.HashedName), // ignore since wrong type - fmt.Sprintf("add %s 1.2.3.4,tcp", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) - }, - }, - expectedLines: []string{ - // TODO ideally we shouldn't create this set because the line will fail in the first try for ipset restore - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - "", - }, - }, - { - name: "ignore after add with bad parent", - args: args{ - []*IPSetMetadata{TestKeyPodSet.Metadata}, - []string{ - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include this - fmt.Sprintf("add %s 7.7.7.7", TestKeyPodSet.HashedName), // include this add (will DELETE this member) - fmt.Sprintf("add %s 8.8.8.8", TestNSSet.HashedName), // ignore this and jump to next create since it's an unexpected set (will NO-OP [no delete]) - fmt.Sprintf("add %s 9.9.9.9", TestKeyPodSet.HashedName), // ignore add because of error above (will NO-OP [no delete]) - }, - }, - expectedLines: []string{ - fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - fmt.Sprintf("-D %s 7.7.7.7", TestKeyPodSet.HashedName), - "", - }, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileString := strings.Join(tt.args.saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - iMgr.CreateIPSets(tt.args.dirtySet) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - sortedExpectedLines := testAndSortRestoreFileLines(t, tt.expectedLines) - - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err, "ipset restore should be successful") - require.False(t, wasFileAltered, "file should not be altered") - }) - } -} - -func TestFailureOnCreateForNewSet(t *testing.T) { - // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order - // test logic: - // - delete a set - // - create three sets, each with two members. the second set to appear will fail to be created - errorLineNum := 2 - setToCreateAlreadyExistsCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), - ExitCode: 1, - } - calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - // add all of these members to the kernel - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member - iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) - - // get original creator and run it the first time - creator := iMgr.fileCreatorForApply(len(calls), nil) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated - removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - originalLength := len(expectedLines) - expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") - require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnCreateForSetInKernel(t *testing.T) { - // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order - // test logic: - // - delete a set - // - update three sets already in the kernel, each with a delete and add line. the second set to appear will fail to be created - errorLineNum := 2 - setToCreateAlreadyExistsCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), - ExitCode: 1, - } - calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete - fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestKeyPodSet.HashedName), // delete - fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestKVPodSet.HashedName), // delete - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - // add all of these members to the kernel - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "6.7.8.9", "a")) // add member to kernel - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel - iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) - - // get original creator and run it the first time - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated - removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKeyPodSet.HashedName, TestKVPodSet.HashedName}) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - originalLength := len(expectedLines) - expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-D") - require.Equal(t, originalLength-1, len(expectedLines), "expected to remove a delete line") - expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") - require.Equal(t, originalLength-2, len(expectedLines), "expected to remove an add line") - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnAddToListInKernel(t *testing.T) { - // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order - // test logic: - // - delete a set - // - update three lists already in the set, each with a delete and add line. the second list to appear will have the failed add - // - create a set and add a member to it - errorLineNum := 8 - memberDoesNotExistCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel - ExitCode: 1, - } - calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), - fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // delete this member - fmt.Sprintf(createListFormat, TestKVNSList.HashedName), - fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestNSSet.HashedName), // delete this member - fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), - fmt.Sprintf("add %s %s", TestNestedLabelList.HashedName, TestNSSet.HashedName), // delete this member - - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // create and add member - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - // rerun the creator after removing previously run lines, and aborting the member-add line that failed - removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) - removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) - require.Equal(t, TestKeyPodSet.HashedName, removedMember) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnAddToNewList(t *testing.T) { - // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order - // test logic: - // - delete a set - // - update a set already in the kernel with a delete and add line - // - create three lists in the set, each with an add line. the second list to appear will have the failed add - errorLineNum := 8 - memberDoesNotExistCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel - ExitCode: 1, - } - calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete this member - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "a")) // create and add member - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel - require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - // rerun the creator after removing previously run lines, and aborting the member-add line that failed - removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) - removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) - require.Equal(t, TestNSSet.HashedName, removedMember) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnDelete(t *testing.T) { - // TODO -} - -func TestFailureOnFlush(t *testing.T) { - // test logic: - // - delete two sets. the first to appear will fail to flush - // - update a set by deleting a member - // - create a set with a member - errorLineNum := 5 - setDoesNotExistCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: The set with the given name does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel - ExitCode: 1, - } - calls := []testutils.TestCmd{setDoesNotExistCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member - fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - // rerun the creator after aborting the flush and delete for the set that failed to flush - removedSetName := hashedNameOfSetImpacted(t, "-F", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - originalLength := len(expectedLines) - expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-X") - require.Equal(t, originalLength-1, len(expectedLines), "expected to remove one destroy line") - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnDestroy(t *testing.T) { - // test logic: - // - delete two sets. the first to appear will fail to delete - // - update a set by deleting a member - // - create a set with a member - errorLineNum := 7 - inUseByKernelCommand := testutils.TestCmd{ - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: Set cannot be destroyed: it is in use by a kernel component", errorLineNum), - ExitCode: 1, - } - calls := []testutils.TestCmd{inUseByKernelCommand, fakeRestoreSuccessCommand} - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - saveFileLines := []string{ - fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), - fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member - fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member - } - saveFileString := strings.Join(saveFileLines, "\n") - saveFileBytes := []byte(saveFileString) - - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) - originalLines := strings.Split(creator.ToString(), "\n") - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - removedSetName := hashedNameOfSetImpacted(t, "-X", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) - expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run - sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func TestFailureOnLastLine(t *testing.T) { - // make sure that the file recovers and returns no error when there are no more lines on the second run - // test logic: - // - delete a set - errorLineNum := 2 - calls := []testutils.TestCmd{ - { - Cmd: ipsetRestoreStringSlice, - Stdout: fmt.Sprintf("Error in line %d: some destroy error", errorLineNum), - ExitCode: 1, - }, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - - creator := iMgr.fileCreatorForApply(2, nil) - wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - require.Error(t, err, "ipset restore should fail") - require.True(t, wasFileAltered, "file should be altered") - - expectedLines := []string{""} // skip the error line and the lines previously run - actualLines := testAndSortRestoreFileString(t, creator.ToString()) - dptestutils.AssertEqualLines(t, expectedLines, actualLines) - wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") - require.NoError(t, err) - require.False(t, wasFileAltered, "file should not be altered") -} - -func testAndSortRestoreFileString(t *testing.T, multilineString string) []string { - return testAndSortRestoreFileLines(t, strings.Split(multilineString, "\n")) -} - -// make sure file goes in order of creates, adds/deletes, flushes, then destroys -// then sort those sections and return the lines in an array -func testAndSortRestoreFileLines(t *testing.T, lines []string) []string { - if len(lines) == 0 { - return lines - } - require.True(t, lines[len(lines)-1] == "", "restore file must end with blank line") - lines = lines[:len(lines)-1] // remove the blank line - - // order of operation groups in restore file (can have groups with multiple possible operatoins) - operationGroups := [][]string{ - {"-N"}, // creates - {"-A", "-D"}, // adds/deletes - {"-F"}, // flushes - {"-X"}, // destroys - } - result := make([]string, 0, len(lines)) - groupIndex := 0 - groupStartIndex := 0 - k := 0 - for k < len(lines) { - for k < len(lines) { - // iterate until we reach an operation not in the current operation group - operation := lines[k][0:2] - expectedOperations := operationGroups[groupIndex] - if !isStringInSlice(operation, expectedOperations) { - require.True(t, groupIndex < len(operationGroups)-1, "ran out of operation groups. got operation %s", operation) - operationLines := lines[groupStartIndex:k] - sort.Strings(operationLines) - result = append(result, operationLines...) - groupStartIndex = k - groupIndex++ - break - } - k++ - } - } - // add the remaining lines since the final operation group won't pass through the if statement in the loop above - operatrionLines := lines[groupStartIndex:] - sort.Strings(operatrionLines) - result = append(result, operatrionLines...) - result = append(result, "") // add the blank line - return result -} - -func hashedNameOfSetImpacted(t *testing.T, operation string, lines []string, lineNum int) string { - lineNumIndex := lineNum - 1 - line := lines[lineNumIndex] - pattern := fmt.Sprintf(`\%s (azure-npm-\d+)`, operation) - re := regexp.MustCompile(pattern) - results := re.FindStringSubmatch(line) - require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) - return results[1] // second item in slice is the group surrounded by () -} - -func memberNameOfSetImpacted(t *testing.T, lines []string, lineNum int) string { - lineNumIndex := lineNum - 1 - line := lines[lineNumIndex] - pattern := `\-[AD] azure-npm-\d+ (.*)` - re := regexp.MustCompile(pattern) - member := re.FindStringSubmatch(line)[1] - results := re.FindStringSubmatch(line) - require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) - return member -} - -func isStringInSlice(item string, values []string) bool { - success := false - for _, value := range values { - if item == value { - success = true - break - } - } - return success -} - -func requireStringInSlice(t *testing.T, item string, values []string) { - require.Truef(t, isStringInSlice(item, values), "item %s was not one of the possible values %+v", item, values) -} - -// remove lines that start with the operation (include the dash in the operations) e.g. -// -A 1.2.3.4 -// -D 1.2.3.4 -// -X -func removeOperationsForSet(lines []string, hashedSetName, operation string) []string { - operationRegex := regexp.MustCompile(fmt.Sprintf(`\%s %s`, operation, hashedSetName)) - goodLines := []string{} - for _, line := range lines { - if !operationRegex.MatchString(line) { - goodLines = append(goodLines, line) - } - } - return goodLines -} +// FIXME uncomment +// func TestApplyIPSets(t *testing.T) { +// type args struct { +// toAddUpdateSets []*IPSetMetadata +// toDeleteSets []*IPSetMetadata +// commandError bool +// } +// tests := []struct { +// name string +// args args +// expectedExecCount int +// wantErr bool +// }{ +// { +// name: "nothing to update", +// args: args{ +// toAddUpdateSets: nil, +// toDeleteSets: nil, +// commandError: false, +// }, +// expectedExecCount: 0, +// wantErr: false, +// }, +// { +// name: "success with just add", +// args: args{ +// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, +// toDeleteSets: nil, +// commandError: false, +// }, +// expectedExecCount: 1, +// wantErr: false, +// }, +// { +// name: "success with just delete", +// args: args{ +// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, +// toDeleteSets: nil, +// commandError: false, +// }, +// expectedExecCount: 1, +// wantErr: false, +// }, +// { +// name: "success with add and delete", +// args: args{ +// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, +// toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, +// commandError: false, +// }, +// expectedExecCount: 1, +// wantErr: false, +// }, +// { +// name: "set is in both delete and add/update cache", +// args: args{ +// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, +// toDeleteSets: []*IPSetMetadata{namespaceSet}, +// commandError: false, +// }, +// expectedExecCount: 1, +// wantErr: false, +// }, +// { +// name: "apply error", +// args: args{ +// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, +// toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, +// commandError: true, +// }, +// expectedExecCount: 1, +// wantErr: true, +// }, +// } +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// metrics.ReinitializeAll() +// calls := GetApplyIPSetsTestCalls(tt.args.toAddUpdateSets, tt.args.toDeleteSets) +// if tt.args.commandError { +// // add an error to the last call (the ipset-restore call) +// // this would potentially cause problems if we used pointers to the TestCmds +// require.Greater(t, len(calls), 0) +// calls[len(calls)-1].ExitCode = 1 +// // then add errors as many times as we retry +// for i := 1; i < maxTryCount; i++ { +// calls = append(calls, testutils.TestCmd{Cmd: ipsetRestoreStringSlice, ExitCode: 1}) +// } +// } +// ioShim := common.NewMockIOShim(calls) +// defer ioShim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) +// iMgr.CreateIPSets(tt.args.toAddUpdateSets) +// for _, set := range tt.args.toDeleteSets { +// iMgr.dirtyCache.delete(NewIPSet(set)) +// } +// err := iMgr.ApplyIPSets() + +// // cache behavior is currently undefined if there's an apply error +// if tt.wantErr { +// require.Error(t, err) +// } else { +// require.NoError(t, err) +// cache := make([]setMembers, 0) +// for _, set := range tt.args.toAddUpdateSets { +// cache = append(cache, setMembers{metadata: set, members: nil}) +// } +// assertExpectedInfo(t, iMgr, &expectedInfo{ +// mainCache: cache, +// toAddUpdateCache: nil, +// toDeleteCache: nil, +// setsForKernel: nil, +// }) +// } + +// execCount, err := metrics.GetIPSetExecCount() +// promutil.NotifyIfErrors(t, err) +// require.Equal(t, tt.expectedExecCount, execCount) +// }) +// } +// } + +// func TestDestroyNPMIPSetsCreatorSuccess(t *testing.T) { +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) +// creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) +// actualLines := strings.Split(creator.ToString(), "\n") +// expectedLines := []string{ +// "-F azure-npm-123456", +// "-F azure-npm-987654", +// "-F azure-npm-777777", +// "-X azure-npm-123456", +// "-X azure-npm-987654", +// "-X azure-npm-777777", +// "", +// } +// dptestutils.AssertEqualLines(t, expectedLines, actualLines) +// require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") +// wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.False(t, wasModified) +// require.NoError(t, err) +// require.Equal(t, 0, *destroyFailureCount, "got unexpected failure count") +// } + +// func TestDestroyNPMIPSetsCreatorErrorHandling(t *testing.T) { +// tests := []struct { +// name string +// call testutils.TestCmd +// expectedLines []string +// expectedFailureCount int +// }{ +// { +// name: "set doesn't exist on flush", +// call: testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 2: The set with the given name does not exist", +// ExitCode: 1, +// }, +// expectedLines: []string{ +// "-F azure-npm-777777", +// "-X azure-npm-123456", +// "-X azure-npm-777777", +// "", +// }, +// expectedFailureCount: 0, +// }, +// { +// name: "some other error on flush", +// call: testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 2: for some other error", +// ExitCode: 1, +// }, +// expectedLines: []string{ +// "-F azure-npm-777777", +// "-X azure-npm-123456", +// "-X azure-npm-777777", +// "", +// }, +// expectedFailureCount: 1, +// }, +// { +// name: "set doesn't exist on destroy", +// call: testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 5: The set with the given name does not exist", +// ExitCode: 1, +// }, +// expectedLines: []string{ +// "-X azure-npm-777777", +// "", +// }, +// expectedFailureCount: 0, +// }, +// { +// name: "some other error on destroy", +// call: testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 5: some other error", +// ExitCode: 1, +// }, +// expectedLines: []string{ +// "-X azure-npm-777777", +// "", +// }, +// expectedFailureCount: 1, +// }, +// } +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// calls := []testutils.TestCmd{tt.call} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) +// creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) +// require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") +// wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.True(t, wasModified) +// require.Error(t, err) +// actualLines := strings.Split(creator.ToString(), "\n") +// dptestutils.AssertEqualLines(t, tt.expectedLines, actualLines) +// require.Equal(t, tt.expectedFailureCount, *destroyFailureCount, "got unexpected failure count") +// }) +// } +// } + +// func TestDestroyNPMIPSets(t *testing.T) { +// tests := []struct { +// name string +// calls []testutils.TestCmd +// wantErr bool +// }{ +// { +// name: "success with no results from grep", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, ExitCode: 1}, +// }, +// wantErr: false, +// }, +// { +// name: "successfully delete sets", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// fakeRestoreSuccessCommand, +// }, +// wantErr: false, +// }, +// { +// name: "grep error", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, HasStartError: true, PipedToCommand: true, ExitCode: 1}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// }, +// wantErr: true, +// }, +// { +// name: "restore error from max tries", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// }, +// wantErr: true, +// }, +// { +// name: "successfully restore, but fail to flush/destroy 1 set since the set doesn't exist when flushing", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// { +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 2: The set with the given name does not exist", +// ExitCode: 1, +// }, +// fakeRestoreSuccessCommand, +// }, +// wantErr: false, +// }, +// { +// name: "successfully restore, but fail to flush/destroy 1 set due to other flush error", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// { +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 2: for some other error", +// ExitCode: 1, +// }, +// fakeRestoreSuccessCommand, +// }, +// wantErr: false, +// }, +// { +// name: "successfully restore, but fail to destroy 1 set since the set doesn't exist when destroying", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// { +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 5: The set with the given name does not exist", +// ExitCode: 1, +// }, +// fakeRestoreSuccessCommand, +// }, +// wantErr: false, +// }, +// { +// name: "successfully restore, but fail to destroy 1 set due to other destroy error", +// calls: []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, +// { +// Cmd: ipsetRestoreStringSlice, +// Stdout: "Error in line 5: for some other error", +// ExitCode: 1, +// }, +// fakeRestoreSuccessCommand, +// }, +// wantErr: false, +// }, +// } + +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// ioshim := common.NewMockIOShim(tt.calls) +// defer ioshim.VerifyCalls(t, tt.calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) +// err := iMgr.resetIPSets() +// if tt.wantErr { +// require.Error(t, err) +// } else { +// require.NoError(t, err) +// } +// }) +// } +// } + +// // identical to TestResetIPSets in ipsetmanager_test.go except an error occurs +// // makes sure that the cache and metrics are reset despite error +// func TestResetIPSetsOnFailure(t *testing.T) { +// metrics.ReinitializeAll() +// calls := []testutils.TestCmd{ +// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true, HasStartError: true}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// } +// ioShim := common.NewMockIOShim(calls) +// defer ioShim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) + +// iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet, keyLabelOfPodSet}) + +// metrics.IncNumIPSets() +// metrics.IncNumIPSets() +// metrics.AddEntryToIPSet("test1") +// metrics.AddEntryToIPSet("test1") +// metrics.AddEntryToIPSet("test2") + +// require.NoError(t, iMgr.ResetIPSets()) + +// assertExpectedInfo(t, iMgr, &expectedInfo{ +// mainCache: nil, +// toAddUpdateCache: nil, +// toDeleteCache: nil, +// setsForKernel: nil, +// }) +// } + +// func TestApplyIPSetsSuccessWithoutSave(t *testing.T) { +// // no sets to add/update, so don't call ipset save +// calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // delete a set so the file isn't empty (otherwise the creator won't even call the exec command) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestNSSet.PrefixName, util.SoftDelete) +// err := iMgr.applyIPSets() +// require.NoError(t, err) +// } + +// func TestApplyIPSetsSuccessWithSave(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// {Cmd: ipsetRestoreStringSlice}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // create a set so we run ipset save +// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) +// err := iMgr.applyIPSets() +// require.NoError(t, err) +// } + +// func TestApplyIPSetsFailureOnSave(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, HasStartError: true, PipedToCommand: true, ExitCode: 1}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // create a set so we run ipset save +// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) +// err := iMgr.applyIPSets() +// require.Error(t, err) +// } + +// func TestApplyIPSetsFailureOnRestore(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// // fail 3 times because this is our max try count +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // create a set so we run ipset save +// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) +// err := iMgr.applyIPSets() +// require.Error(t, err) +// } + +// func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, +// {Cmd: ipsetRestoreStringSlice}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // create a set so we run ipset save +// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) +// err := iMgr.applyIPSets() +// require.NoError(t, err) +// } + +// func TestIPSetSave(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, +// {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// output, err := iMgr.ipsetSave() +// require.NoError(t, err) +// require.Equal(t, saveResult, string(output)) +// } + +// func TestIPSetSaveNoMatch(t *testing.T) { +// calls := []testutils.TestCmd{ +// {Cmd: ipsetSaveStringSlice, ExitCode: 1}, +// {Cmd: []string{"grep", "azure-npm-"}}, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// output, err := iMgr.ipsetSave() +// require.NoError(t, err) +// require.Nil(t, output) +// } + +// func TestCreateForAllSetTypes(t *testing.T) { +// // without save file +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) +// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) + +// creator := iMgr.fileCreatorForApply(len(calls), nil) +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) + +// expectedLines := []string{ +// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), +// fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), +// fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), +// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), +// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), +// fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), +// "", +// } +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err, "ipset restore should be successful") +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestDestroy(t *testing.T) { +// // without save file +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // remove some members and destroy some sets +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) +// require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) +// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) +// require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), nil) +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) + +// expectedLines := []string{ +// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), +// fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), +// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), +// fmt.Sprintf("-F %s", TestCIDRSet.HashedName), +// fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), +// fmt.Sprintf("-X %s", TestCIDRSet.HashedName), +// fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), +// "", +// } +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err, "ipset restore should be successful") +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnCreateForNewSet(t *testing.T) { +// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order +// // test logic: +// // - delete a set +// // - create three sets, each with two members. the second set to appear will fail to be created +// errorLineNum := 2 +// setToCreateAlreadyExistsCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// // add all of these members to the kernel +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member +// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) + +// // get original creator and run it the first time +// creator := iMgr.fileCreatorForApply(len(calls), nil) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated +// removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// originalLength := len(expectedLines) +// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") +// require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnCreateForSetInKernel(t *testing.T) { +// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order +// // test logic: +// // - delete a set +// // - update three sets already in the kernel, each with a delete and add line. the second set to appear will fail to be created +// errorLineNum := 2 +// setToCreateAlreadyExistsCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestKeyPodSet.HashedName), // delete +// fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestKVPodSet.HashedName), // delete +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// // add all of these members to the kernel +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "6.7.8.9", "a")) // add member to kernel +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel +// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) + +// // get original creator and run it the first time +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated +// removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKeyPodSet.HashedName, TestKVPodSet.HashedName}) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// originalLength := len(expectedLines) +// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-D") +// require.Equal(t, originalLength-1, len(expectedLines), "expected to remove a delete line") +// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") +// require.Equal(t, originalLength-2, len(expectedLines), "expected to remove an add line") +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnAddToListInKernel(t *testing.T) { +// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order +// // test logic: +// // - delete a set +// // - update three lists already in the set, each with a delete and add line. the second list to appear will have the failed add +// // - create a set and add a member to it +// errorLineNum := 8 +// memberDoesNotExistCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), +// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // delete this member +// fmt.Sprintf(createListFormat, TestKVNSList.HashedName), +// fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestNSSet.HashedName), // delete this member +// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), +// fmt.Sprintf("add %s %s", TestNestedLabelList.HashedName, TestNSSet.HashedName), // delete this member + +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // create and add member +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// // rerun the creator after removing previously run lines, and aborting the member-add line that failed +// removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) +// removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) +// require.Equal(t, TestKeyPodSet.HashedName, removedMember) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnAddToNewList(t *testing.T) { +// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order +// // test logic: +// // - delete a set +// // - update a set already in the kernel with a delete and add line +// // - create three lists in the set, each with an add line. the second list to appear will have the failed add +// errorLineNum := 8 +// memberDoesNotExistCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete this member +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "a")) // create and add member +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// // rerun the creator after removing previously run lines, and aborting the member-add line that failed +// removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) +// removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) +// require.Equal(t, TestNSSet.HashedName, removedMember) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnDelete(t *testing.T) { +// // TODO +// } + +// func TestFailureOnFlush(t *testing.T) { +// // test logic: +// // - delete two sets. the first to appear will fail to flush +// // - update a set by deleting a member +// // - create a set with a member +// errorLineNum := 5 +// setDoesNotExistCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: The set with the given name does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{setDoesNotExistCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member +// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet +// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// // rerun the creator after aborting the flush and delete for the set that failed to flush +// removedSetName := hashedNameOfSetImpacted(t, "-F", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// originalLength := len(expectedLines) +// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-X") +// require.Equal(t, originalLength-1, len(expectedLines), "expected to remove one destroy line") +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnDestroy(t *testing.T) { +// // test logic: +// // - delete two sets. the first to appear will fail to delete +// // - update a set by deleting a member +// // - create a set with a member +// errorLineNum := 7 +// inUseByKernelCommand := testutils.TestCmd{ +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: Set cannot be destroyed: it is in use by a kernel component", errorLineNum), +// ExitCode: 1, +// } +// calls := []testutils.TestCmd{inUseByKernelCommand, fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member +// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet +// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// originalLines := strings.Split(creator.ToString(), "\n") +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// removedSetName := hashedNameOfSetImpacted(t, "-X", originalLines, errorLineNum) +// requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) +// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestFailureOnLastLine(t *testing.T) { +// // make sure that the file recovers and returns no error when there are no more lines on the second run +// // test logic: +// // - delete a set +// errorLineNum := 2 +// calls := []testutils.TestCmd{ +// { +// Cmd: ipsetRestoreStringSlice, +// Stdout: fmt.Sprintf("Error in line %d: some destroy error", errorLineNum), +// ExitCode: 1, +// }, +// } +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(2, nil) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.Error(t, err, "ipset restore should fail") +// require.True(t, wasFileAltered, "file should be altered") + +// expectedLines := []string{""} // skip the error line and the lines previously run +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// dptestutils.AssertEqualLines(t, expectedLines, actualLines) +// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err) +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func testAndSortRestoreFileString(t *testing.T, multilineString string) []string { +// return testAndSortRestoreFileLines(t, strings.Split(multilineString, "\n")) +// } + +// // make sure file goes in order of creates, adds/deletes, flushes, then destroys +// // then sort those sections and return the lines in an array +// func testAndSortRestoreFileLines(t *testing.T, lines []string) []string { +// if len(lines) == 0 { +// return lines +// } +// require.True(t, lines[len(lines)-1] == "", "restore file must end with blank line") +// lines = lines[:len(lines)-1] // remove the blank line + +// // order of operation groups in restore file (can have groups with multiple possible operatoins) +// operationGroups := [][]string{ +// {"-N"}, // creates +// {"-A", "-D"}, // adds/deletes +// {"-F"}, // flushes +// {"-X"}, // destroys +// } +// result := make([]string, 0, len(lines)) +// groupIndex := 0 +// groupStartIndex := 0 +// k := 0 +// for k < len(lines) { +// for k < len(lines) { +// // iterate until we reach an operation not in the current operation group +// operation := lines[k][0:2] +// expectedOperations := operationGroups[groupIndex] +// if !isStringInSlice(operation, expectedOperations) { +// require.True(t, groupIndex < len(operationGroups)-1, "ran out of operation groups. got operation %s", operation) +// operationLines := lines[groupStartIndex:k] +// sort.Strings(operationLines) +// result = append(result, operationLines...) +// groupStartIndex = k +// groupIndex++ +// break +// } +// k++ +// } +// } +// // add the remaining lines since the final operation group won't pass through the if statement in the loop above +// operatrionLines := lines[groupStartIndex:] +// sort.Strings(operatrionLines) +// result = append(result, operatrionLines...) +// result = append(result, "") // add the blank line +// return result +// } + +// func hashedNameOfSetImpacted(t *testing.T, operation string, lines []string, lineNum int) string { +// lineNumIndex := lineNum - 1 +// line := lines[lineNumIndex] +// pattern := fmt.Sprintf(`\%s (azure-npm-\d+)`, operation) +// re := regexp.MustCompile(pattern) +// results := re.FindStringSubmatch(line) +// require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) +// return results[1] // second item in slice is the group surrounded by () +// } + +// func memberNameOfSetImpacted(t *testing.T, lines []string, lineNum int) string { +// lineNumIndex := lineNum - 1 +// line := lines[lineNumIndex] +// pattern := `\-[AD] azure-npm-\d+ (.*)` +// re := regexp.MustCompile(pattern) +// member := re.FindStringSubmatch(line)[1] +// results := re.FindStringSubmatch(line) +// require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) +// return member +// } + +// func isStringInSlice(item string, values []string) bool { +// success := false +// for _, value := range values { +// if item == value { +// success = true +// break +// } +// } +// return success +// } + +// func requireStringInSlice(t *testing.T, item string, values []string) { +// require.Truef(t, isStringInSlice(item, values), "item %s was not one of the possible values %+v", item, values) +// } + +// // remove lines that start with the operation (include the dash in the operations) e.g. +// // -A 1.2.3.4 +// // -D 1.2.3.4 +// // -X +// func removeOperationsForSet(lines []string, hashedSetName, operation string) []string { +// operationRegex := regexp.MustCompile(fmt.Sprintf(`\%s %s`, operation, hashedSetName)) +// goodLines := []string{} +// for _, line := range lines { +// if !operationRegex.MatchString(line) { +// goodLines = append(goodLines, line) +// } +// } +// return goodLines +// } + +// FIXME LEAVE COMMENTED FOR LATER +// func TestNextCreateLine(t *testing.T) { +// createLine := "create test-list1 list:set size 8" +// addLine := "add test-set1 1.2.3.4" +// createLineWithNewline := createLine + "\n" +// addLineWithNewline := addLine + "\n" +// tests := []struct { +// name string +// lines []string +// expectedReadIndex int +// expectedLine []byte +// }{ +// // parse.Line will omit the newline at the end of the line unless it's the last line +// { +// name: "empty save file", +// lines: []string{}, +// expectedReadIndex: 0, +// expectedLine: nil, +// }, +// { +// name: "no creates", +// lines: []string{addLineWithNewline}, +// expectedReadIndex: len(addLineWithNewline), +// expectedLine: []byte(addLineWithNewline), +// }, +// { +// name: "start with create", +// lines: []string{createLine, addLineWithNewline}, +// expectedReadIndex: len(createLineWithNewline), +// expectedLine: []byte(createLine), +// }, +// { +// name: "create after adds", +// lines: []string{addLine, addLine, createLineWithNewline}, +// expectedReadIndex: 2*len(addLine+"\n") + len(createLine+"\n"), +// expectedLine: []byte(createLineWithNewline), +// }, +// } +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// saveFile := []byte(strings.Join(tt.lines, "\n")) +// line, readIndex := nextCreateLine(0, saveFile) +// require.Equal(t, tt.expectedReadIndex, readIndex) +// require.Equal(t, tt.expectedLine, line) +// }) +// } +// // fmt.Println(string([]byte(addLine + addLine, createLineWithNewline})[:78])) +// } + +// func TestHaveTypeProblem(t *testing.T) { +// type args struct { +// metadata *IPSetMetadata +// format string +// } +// tests := []struct { +// name string +// args args +// wantProblem bool +// }{ +// { +// name: "correct type for nethash", +// args: args{ +// TestNSSet.Metadata, +// createNethashFormat, +// }, +// wantProblem: false, +// }, +// { +// name: "nethash instead of porthash", +// args: args{ +// TestNamedportSet.Metadata, +// createNethashFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "nethash instead of list", +// args: args{ +// TestKeyNSList.Metadata, +// createNethashFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "correct type for porthash", +// args: args{ +// TestNamedportSet.Metadata, +// createPorthashFormat, +// }, +// wantProblem: false, +// }, +// { +// name: "porthash instead of nethash", +// args: args{ +// TestNSSet.Metadata, +// createPorthashFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "porthash instead of list", +// args: args{ +// TestKeyNSList.Metadata, +// createPorthashFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "correct type for list", +// args: args{ +// TestKeyNSList.Metadata, +// createListFormat, +// }, +// wantProblem: false, +// }, +// { +// name: "list instead of nethash", +// args: args{ +// TestNSSet.Metadata, +// createListFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "list instead of porthash", +// args: args{ +// TestNamedportSet.Metadata, +// createListFormat, +// }, +// wantProblem: true, +// }, +// { +// name: "unknown type", +// args: args{ +// TestKeyNSList.Metadata, +// "create %s unknown-type", +// }, +// wantProblem: true, +// }, +// { +// name: "no rest of line", +// args: args{ +// TestKeyNSList.Metadata, +// "create %s", +// }, +// wantProblem: true, +// }, +// } +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// set := NewIPSet(tt.args.metadata) +// line := fmt.Sprintf(tt.args.format, set.HashedName) +// splitLine := strings.Split(line, " ") +// restOfLine := splitLine[2:] +// if tt.wantProblem { +// require.True(t, haveTypeProblem(set, restOfLine)) +// } else { +// require.False(t, haveTypeProblem(set, restOfLine)) +// } +// }) +// } +// } + +// func TestUpdateWithIdenticalSaveFile(t *testing.T) { +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), +// fmt.Sprintf("add %s 10.0.0.5", TestKeyPodSet.HashedName), +// fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), +// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), +// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), +// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), +// fmt.Sprintf(createListFormat, TestKVNSList.HashedName), +// fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), +// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) +// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) + +// expectedLines := []string{ +// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), +// fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), +// "", +// } +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err, "ipset restore should be successful") +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestUpdateWithRealisticSaveFile(t *testing.T) { +// // save file doesn't have some sets we're adding and has some sets that: +// // - aren't dirty +// // - will be deleted +// // - have members which we will delete +// // - are missing members, which we will add +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileLines := []string{ +// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), // should add 10.0.0.1-5 to this set +// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member +// fmt.Sprintf("add %s 5.6.7.8", TestNSSet.HashedName), // delete this member +// fmt.Sprintf("add %s 5.6.7.9", TestNSSet.HashedName), // delete this member +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // dirty but no member changes in the end +// fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), // ignore this set since it's not dirty +// fmt.Sprintf("add %s 1.2.3.4", TestKVPodSet.HashedName), // ignore this set since it's not dirty +// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), // should add TestKeyPodSet to this set +// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // keep this member +// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), // delete this member +// fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), // ignore this set since it's not dirty +// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), // this set will be deleted +// } +// saveFileString := strings.Join(saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.2", "c")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.3", "d")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.4", "e")) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.5", "f")) +// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) +// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) +// iMgr.CreateIPSets([]*IPSetMetadata{TestKVNSList.Metadata}) +// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "z")) // set not in save file +// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete +// iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) // adding NSSet and KeyPodSet (should be keeping NSSet and deleting NamedportSet) + +// expectedLines := []string{ +// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), +// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), +// fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), +// fmt.Sprintf("-A %s 1.2.3.4", TestCIDRSet.HashedName), +// fmt.Sprintf("-D %s 5.6.7.8", TestNSSet.HashedName), +// fmt.Sprintf("-D %s 5.6.7.9", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.2", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.3", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.4", TestNSSet.HashedName), +// fmt.Sprintf("-A %s 10.0.0.5", TestNSSet.HashedName), +// fmt.Sprintf("-D %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), +// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), +// fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), +// fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), +// "", +// } +// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err, "ipset restore should be successful") +// require.False(t, wasFileAltered, "file should not be altered") +// } + +// func TestUpdateWithBadSaveFile(t *testing.T) { +// type args struct { +// dirtySet []*IPSetMetadata +// saveFileLines []string +// } +// tests := []struct { +// name string +// args args +// expectedLines []string +// }{ +// { +// name: "no create line", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), +// fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), +// }, +// }, +// expectedLines: []string{ +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// { +// name: "unexpected verb after create", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), +// "wrong-verb ...", +// }, +// }, +// expectedLines: []string{ +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// { +// name: "non-NPM set", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// "create test-set1 hash:net family inet hashsize 1024 maxelem 65536", +// "add test-set1 1.2.3.4", +// }, +// }, +// expectedLines: []string{ +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// { +// name: "ignore set we've already parsed", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include +// fmt.Sprintf("add %s 4.4.4.4", TestKeyPodSet.HashedName), // include this add (will DELETE this member) +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // ignore this create and ensuing adds since we already included this set +// fmt.Sprintf("add %s 5.5.5.5", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) +// }, +// }, +// expectedLines: []string{ +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-D %s 4.4.4.4", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// { +// name: "set with wrong type", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// fmt.Sprintf(createPorthashFormat, TestKeyPodSet.HashedName), // ignore since wrong type +// fmt.Sprintf("add %s 1.2.3.4,tcp", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) +// }, +// }, +// expectedLines: []string{ +// // TODO ideally we shouldn't create this set because the line will fail in the first try for ipset restore +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// { +// name: "ignore after add with bad parent", +// args: args{ +// []*IPSetMetadata{TestKeyPodSet.Metadata}, +// []string{ +// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include this +// fmt.Sprintf("add %s 7.7.7.7", TestKeyPodSet.HashedName), // include this add (will DELETE this member) +// fmt.Sprintf("add %s 8.8.8.8", TestNSSet.HashedName), // ignore this and jump to next create since it's an unexpected set (will NO-OP [no delete]) +// fmt.Sprintf("add %s 9.9.9.9", TestKeyPodSet.HashedName), // ignore add because of error above (will NO-OP [no delete]) +// }, +// }, +// expectedLines: []string{ +// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), +// fmt.Sprintf("-D %s 7.7.7.7", TestKeyPodSet.HashedName), +// "", +// }, +// }, +// } +// for _, tt := range tests { +// tt := tt +// t.Run(tt.name, func(t *testing.T) { +// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} +// ioshim := common.NewMockIOShim(calls) +// defer ioshim.VerifyCalls(t, calls) +// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + +// saveFileString := strings.Join(tt.args.saveFileLines, "\n") +// saveFileBytes := []byte(saveFileString) + +// iMgr.CreateIPSets(tt.args.dirtySet) + +// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) +// actualLines := testAndSortRestoreFileString(t, creator.ToString()) +// sortedExpectedLines := testAndSortRestoreFileLines(t, tt.expectedLines) + +// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) +// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") +// require.NoError(t, err, "ipset restore should be successful") +// require.False(t, wasFileAltered, "file should not be altered") +// }) +// } +// } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go index 3128ad67d6..f5e349e3a3 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go @@ -1392,26 +1392,23 @@ func assertExpectedInfo(t *testing.T, iMgr *IPSetManager, info *expectedInfo) { } // 1.2. make sure the toAddOrUpdateCache is equal - require.Equal(t, len(info.toAddUpdateCache), len(iMgr.toAddOrUpdateCache), "toAddUpdateCache size mismatch") + require.Equal(t, len(info.toAddUpdateCache), iMgr.dirtyCache.numSetsToAddOrUpdate(), "toAddUpdateCache size mismatch") for _, setMetadata := range info.toAddUpdateCache { setName := setMetadata.GetPrefixName() - _, ok := iMgr.toAddOrUpdateCache[setName] - require.True(t, ok, "set %s not in the toAddUpdateCache") + require.True(t, iMgr.dirtyCache.isSetToAddOrUpdate(setName), "set %s not in the toAddUpdateCache") require.True(t, iMgr.exists(setName), "set %s not in the main cache but is in the toAddUpdateCache", setName) } // 1.3. make sure the toDeleteCache is equal - require.Equal(t, len(info.toDeleteCache), len(iMgr.toDeleteCache), "toDeleteCache size mismatch") + require.Equal(t, len(info.toDeleteCache), iMgr.dirtyCache.numSetsToDelete(), "toDeleteCache size mismatch") for _, setName := range info.toDeleteCache { - _, ok := iMgr.toDeleteCache[setName] - require.True(t, ok, "set %s not found in toDeleteCache", setName) + require.True(t, iMgr.dirtyCache.isSetToDelete(setName), "set %s not found in toDeleteCache", setName) } // 1.4. assert kernel status of sets in the toAddOrUpdateCache for _, setMetadata := range info.setsForKernel { // check semantics - _, ok := iMgr.toAddOrUpdateCache[setMetadata.GetPrefixName()] - require.True(t, ok, "setsForKernel should be a subset of toAddUpdateCache") + require.True(t, iMgr.dirtyCache.isSetToAddOrUpdate(setMetadata.GetPrefixName()), "setsForKernel should be a subset of toAddUpdateCache") setName := setMetadata.GetPrefixName() require.True(t, iMgr.exists(setName), "kernel set %s not found", setName) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index 2c3072d9c2..cc8a8b229b 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -124,7 +124,8 @@ func (iMgr *IPSetManager) applyIPSets() error { } } - iMgr.toAddOrUpdateCache = make(map[string]struct{}) + // FIXME uncomment and handle? + // iMgr.toAddOrUpdateCache = make(map[string]struct{}) if len(setPolicyBuilder.toDeleteSets) > 0 { err = iMgr.modifySetPolicies(network, hcn.RequestTypeRemove, setPolicyBuilder.toDeleteSets) @@ -157,7 +158,7 @@ func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkP } existingSets, toDeleteSets := iMgr.segregateSetPolicies(networkPolicies, donotResetIPSets) // some of this below logic can be abstracted a step above - toAddUpdateSetNames := iMgr.toAddOrUpdateCache + toAddUpdateSetNames := iMgr.dirtyCache.getSetsToAddOrUpdate() setPolicyBuilder.toDeleteSets = toDeleteSets // for faster look up changing a slice to map @@ -168,7 +169,7 @@ func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkP // (TODO) remove this log line later klog.Infof("toAddUpdateSetNames %+v \n ", toAddUpdateSetNames) klog.Infof("existingSetNames %+v \n ", existingSetNames) - for setName := range toAddUpdateSetNames { + for _, setName := range toAddUpdateSetNames { set, exists := iMgr.setMap[setName] // check if the Set exists if !exists { return nil, errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("ipset %s does not exist", setName)) @@ -276,7 +277,7 @@ func (iMgr *IPSetManager) segregateSetPolicies(networkPolicies []hcn.NetworkPoli if !strings.HasPrefix(set.Id, util.AzureNpmPrefix) { continue } - _, ok := iMgr.toDeleteCache[set.Name] + ok := iMgr.dirtyCache.isSetToDelete(set.Name) if !ok && !reset { // if the set is not in delete cache, go ahead and add it to update cache toUpdateSets = append(toUpdateSets, set.Name) diff --git a/npm/pkg/dataplane/ipsets/testutils_linux.go b/npm/pkg/dataplane/ipsets/testutils_linux.go index 2189a2d0fb..344bff312b 100644 --- a/npm/pkg/dataplane/ipsets/testutils_linux.go +++ b/npm/pkg/dataplane/ipsets/testutils_linux.go @@ -14,13 +14,7 @@ var ( ) func GetApplyIPSetsTestCalls(toAddOrUpdateIPSets, toDeleteIPSets []*IPSetMetadata) []testutils.TestCmd { - if len(toAddOrUpdateIPSets) > 0 { - return []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, ExitCode: 1}, // grep didn't find anything - {Cmd: ipsetRestoreStringSlice}, - } - } else if len(toDeleteIPSets) == 0 { + if len(toAddOrUpdateIPSets) == 0 && len(toDeleteIPSets) == 0 { return []testutils.TestCmd{} } return []testutils.TestCmd{fakeRestoreSuccessCommand} From 71bf5adf2081afa91700eb01df599dcc29a41101 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 4 Apr 2022 09:16:35 -0700 Subject: [PATCH 02/17] fix member update --- npm/pkg/dataplane/ipsets/ipsetmanager.go | 40 +++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index b8dafb4505..cc42ff45a3 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -231,13 +231,12 @@ func (iMgr *IPSetManager) AddToSets(addToSets []*IPSetMetadata, ip, podKey strin // 2. add ip to the set, and update the pod key _, ok := set.IPPodKey[ip] - set.IPPodKey[ip] = podKey - if ok { - continue + if !ok { + // needs to be called before updating the cache + iMgr.modifyCacheForKernelMemberUpdate(set) + metrics.AddEntryToIPSet(prefixedName) } - - iMgr.modifyCacheForKernelMemberUpdate(set) - metrics.AddEntryToIPSet(prefixedName) + set.IPPodKey[ip] = podKey } return nil } @@ -283,9 +282,11 @@ func (iMgr *IPSetManager) RemoveFromSets(removeFromSets []*IPSetMetadata, ip, po continue } + // needs to be called before updating the cache + iMgr.modifyCacheForKernelMemberUpdate(set) + // update the IP ownership with podkey delete(set.IPPodKey, ip) - iMgr.modifyCacheForKernelMemberUpdate(set) metrics.RemoveEntryFromIPSet(prefixedName) } return nil @@ -323,7 +324,6 @@ func (iMgr *IPSetManager) AddToLists(listMetadatas, setMetadatas []*IPSetMetadat return npmerrors.Errorf(npmerrors.AppendIPSet, false, msg) } - modified := false // 3. add all members to the list for _, memberMetadata := range setMetadatas { memberName := memberMetadata.GetPrefixName() @@ -337,6 +337,10 @@ func (iMgr *IPSetManager) AddToLists(listMetadatas, setMetadatas []*IPSetMetadat } member := iMgr.setMap[memberName] + // needs to be called before updating the cache + // second call, third call, etc. for a given list is idempotent + iMgr.modifyCacheForKernelMemberUpdate(list) + list.MemberIPSets[memberName] = member member.incIPSetReferCount() metrics.AddEntryToIPSet(list.Name) @@ -344,10 +348,6 @@ func (iMgr *IPSetManager) AddToLists(listMetadatas, setMetadatas []*IPSetMetadat if listIsInKernel { iMgr.incKernelReferCountAndModifyCache(member) } - modified = true - } - if modified { - iMgr.modifyCacheForKernelMemberUpdate(list) } } return nil @@ -373,7 +373,6 @@ func (iMgr *IPSetManager) RemoveFromList(listMetadata *IPSetMetadata, setMetadat return npmerrors.Errorf(npmerrors.DeleteIPSet, false, msg) } - modified := false for _, setMetadata := range setMetadatas { memberName := setMetadata.GetPrefixName() if memberName == "" { @@ -388,9 +387,6 @@ func (iMgr *IPSetManager) RemoveFromList(listMetadata *IPSetMetadata, setMetadat // Nested IPSets are only supported for windows // Check if we want to actually use that support if member.Kind != HashSet { - if modified { - iMgr.modifyCacheForKernelMemberUpdate(list) - } msg := fmt.Sprintf("ipset %s is not a hash set and nested list sets are not supported", memberName) metrics.SendErrorLogAndMetric(util.IpsmID, "error: failed to remove from list: %s", msg) return npmerrors.Errorf(npmerrors.DeleteIPSet, false, msg) @@ -401,6 +397,10 @@ func (iMgr *IPSetManager) RemoveFromList(listMetadata *IPSetMetadata, setMetadat continue } + // needs to be called before updating the cache + // second call, third call, etc. for a given list is idempotent + iMgr.modifyCacheForKernelMemberUpdate(list) + delete(list.MemberIPSets, memberName) member.decIPSetReferCount() metrics.RemoveEntryFromIPSet(list.Name) @@ -408,10 +408,6 @@ func (iMgr *IPSetManager) RemoveFromList(listMetadata *IPSetMetadata, setMetadat if listIsInKernel { iMgr.decKernelReferCountAndModifyCache(member) } - modified = true - } - if modified { - iMgr.modifyCacheForKernelMemberUpdate(list) } return nil } @@ -523,9 +519,11 @@ func (iMgr *IPSetManager) decKernelReferCountAndModifyCache(member *IPSet) { } } +// need to call this before updating the set +// can be called multiple times func (iMgr *IPSetManager) modifyCacheForKernelMemberUpdate(set *IPSet) { if iMgr.shouldBeInKernel(set) { - iMgr.dirtyCache.create(set) + iMgr.dirtyCache.update(set) } } From 7b2402162b3d697d2c897747dc483be771b32564 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 4 Apr 2022 12:06:23 -0700 Subject: [PATCH 03/17] fix hashed name bug --- npm/pkg/dataplane/ipsets/dirtycache.go | 4 ++-- npm/pkg/dataplane/ipsets/ipsetmanager_linux.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index b034fa67b1..b29dbe4855 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -22,7 +22,7 @@ type dirtyCacheMaintainer interface { isSetToAddOrUpdate(setName string) bool // isSetToDelete returns true if the set is dirty and should be deleted isSetToDelete(setName string) bool - // getOriginalMembers returns the original members of the set before it was dirty - // members are either IPs, CIDRs, IP-Port pairs, or set names if the parent is a list + // getOriginalMembers returns the original members of the set before it was dirty. + // members are either IPs, CIDRs, IP-Port pairs, or prefixed set names if the parent is a list getOriginalMembers(setName string) map[string]struct{} } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 93c28601ec..5942f97300 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -307,13 +307,13 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) } } } else { - for setName := range set.MemberIPSets { + for setName, set := range set.MemberIPSets { if _, ok := originalMembers[setName]; ok { // remove from members so we don't try to delete it later delete(originalMembers, setName) } else { // add the member since it didn't exist before - iMgr.addMemberForApply(creator, set, sectionID, setName) + iMgr.addMemberForApply(creator, set, sectionID, set.HashedName) } } } From c6ed3fb05dd5dd395c55ccfd38907966e3f9cc21 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 4 Apr 2022 16:57:36 -0700 Subject: [PATCH 04/17] fix hashed name bug and bug for original members for delete cache --- npm/pkg/dataplane/ipsets/dirtycache_linux.go | 5 ++++- npm/pkg/dataplane/ipsets/ipsetmanager_linux.go | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index f663412db3..b16b54aa71 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -112,7 +112,10 @@ func (dc *memberAwareDirtyCache) isSetToDelete(setName string) bool { func (dc *memberAwareDirtyCache) getOriginalMembers(setName string) map[string]struct{} { info, ok := dc.toAddOrUpdateCache[setName] if !ok { - return nil + info, ok = dc.toDeleteCache[setName] + if !ok { + return nil + } } members := make(map[string]struct{}, len(info.members)) for member := range info.members { diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 5942f97300..a842171154 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -307,13 +307,13 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) } } } else { - for setName, set := range set.MemberIPSets { - if _, ok := originalMembers[setName]; ok { + for memberName, memberSet := range set.MemberIPSets { + if _, ok := originalMembers[memberName]; ok { // remove from members so we don't try to delete it later - delete(originalMembers, setName) + delete(originalMembers, memberName) } else { // add the member since it didn't exist before - iMgr.addMemberForApply(creator, set, sectionID, set.HashedName) + iMgr.addMemberForApply(creator, set, sectionID, memberSet.HashedName) } } } From ebb1a37781de2f41e0fa273d5eb05bc18b45737b Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Tue, 5 Apr 2022 11:54:53 -0700 Subject: [PATCH 05/17] fix delete bug --- npm/pkg/dataplane/ipsets/ipsetmanager_linux.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index a842171154..662fde2c3c 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -317,8 +317,14 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) } } } + // originalMembers now contains only the members that need to be deleted for member := range originalMembers { - iMgr.deleteMemberForApply(creator, set, sectionID, member) + if set.Kind == HashSet { + iMgr.deleteMemberForApply(creator, set, sectionID, member) + } else { + memberSet := iMgr.setMap[member] + iMgr.deleteMemberForApply(creator, set, sectionID, memberSet.HashedName) + } } } From 6f79c8172cc85e82e17380386eb072dc8369b9bd Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 6 Apr 2022 09:59:10 -0700 Subject: [PATCH 06/17] print cache in logs again --- npm/pkg/dataplane/ipsets/dirtycache.go | 4 ++ npm/pkg/dataplane/ipsets/dirtycache_linux.go | 56 +++++++++---------- .../dataplane/ipsets/dirtycache_windows.go | 8 +++ npm/pkg/dataplane/ipsets/ipsetmanager.go | 9 +-- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index b29dbe4855..5f806e3a22 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -22,6 +22,10 @@ type dirtyCacheMaintainer interface { isSetToAddOrUpdate(setName string) bool // isSetToDelete returns true if the set is dirty and should be deleted isSetToDelete(setName string) bool + // printAddOrUpdateCache returns a string representation of the add/update cache + printAddOrUpdateCache() string + // printDeleteCache returns a string representation of the delete cache + printDeleteCache() string // getOriginalMembers returns the original members of the set before it was dirty. // members are either IPs, CIDRs, IP-Port pairs, or prefixed set names if the parent is a list getOriginalMembers(setName string) map[string]struct{} diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index b16b54aa71..ba7cd02a7d 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -1,39 +1,34 @@ package ipsets -type memberAwareDirtyCache struct { - toAddOrUpdateCache map[string]*dirtyInfo - toDeleteCache map[string]*dirtyInfo -} +import "fmt" -type dirtyInfo struct { - setType SetType - members map[string]struct{} +type memberAwareDirtyCache struct { + // map of prefixed set names to original members + toAddOrUpdateCache map[string]map[string]struct{} + // map of prefixed set names to original members + toDeleteCache map[string]map[string]struct{} } func newDirtyCache() dirtyCacheMaintainer { return &memberAwareDirtyCache{ - toAddOrUpdateCache: make(map[string]*dirtyInfo), - toDeleteCache: make(map[string]*dirtyInfo), + toAddOrUpdateCache: make(map[string]map[string]struct{}), + toDeleteCache: make(map[string]map[string]struct{}), } } func (dc *memberAwareDirtyCache) reset() { - dc.toAddOrUpdateCache = make(map[string]*dirtyInfo) - dc.toDeleteCache = make(map[string]*dirtyInfo) + dc.toAddOrUpdateCache = make(map[string]map[string]struct{}) + dc.toDeleteCache = make(map[string]map[string]struct{}) } func (dc *memberAwareDirtyCache) create(newSet *IPSet) { setName := newSet.Name if _, ok := dc.toAddOrUpdateCache[setName]; ok { - // NOTE: could throw error if setType is different return } info, ok := dc.toDeleteCache[setName] if !ok { - info = &dirtyInfo{ - setType: newSet.Type, - members: make(map[string]struct{}), - } + info = make(map[string]struct{}) } dc.toAddOrUpdateCache[setName] = info delete(dc.toDeleteCache, setName) @@ -47,16 +42,15 @@ func (dc *memberAwareDirtyCache) delete(originalSet *IPSet) { putIntoAndRemoveFromOther(originalSet, dc.toDeleteCache, dc.toAddOrUpdateCache) } -func putIntoAndRemoveFromOther(originalSet *IPSet, intoCache, fromCache map[string]*dirtyInfo) { +func putIntoAndRemoveFromOther(originalSet *IPSet, intoCache, fromCache map[string]map[string]struct{}) { setName := originalSet.Name if _, ok := intoCache[setName]; ok { - // NOTE: could throw error if setType is different return } - info, ok := fromCache[setName] + members, ok := fromCache[setName] if !ok { setType := originalSet.Type - members := make(map[string]struct{}) + members = make(map[string]struct{}) if setType.getSetKind() == HashSet { for member := range originalSet.IPPodKey { members[member] = struct{}{} @@ -66,12 +60,8 @@ func putIntoAndRemoveFromOther(originalSet *IPSet, intoCache, fromCache map[stri members[memberName] = struct{}{} } } - info = &dirtyInfo{ - setType: setType, - members: members, - } } - intoCache[setName] = info + intoCache[setName] = members delete(fromCache, setName) } @@ -109,17 +99,21 @@ func (dc *memberAwareDirtyCache) isSetToDelete(setName string) bool { return ok } +func (dc *memberAwareDirtyCache) printAddOrUpdateCache() string { + return fmt.Sprintf("%+v", dc.toAddOrUpdateCache) +} + +func (dc *memberAwareDirtyCache) printDeleteCache() string { + return fmt.Sprintf("%+v", dc.toDeleteCache) +} + func (dc *memberAwareDirtyCache) getOriginalMembers(setName string) map[string]struct{} { - info, ok := dc.toAddOrUpdateCache[setName] + members, ok := dc.toAddOrUpdateCache[setName] if !ok { - info, ok = dc.toDeleteCache[setName] + members, ok = dc.toDeleteCache[setName] if !ok { return nil } } - members := make(map[string]struct{}, len(info.members)) - for member := range info.members { - members[member] = struct{}{} - } return members } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 24f08f7caf..9e646d8abf 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -72,6 +72,14 @@ func (dc *nameOnlyDirtyCache) isSetToDelete(setName string) bool { return ok } +func (dc *nameOnlyDirtyCache) printAddOrUpdateCache() string { + return fmt.Sprintf("%+v", dc.toAddOrUpdateCache) +} + +func (dc *nameOnlyDirtyCache) printDeleteCache() string { + return fmt.Sprintf("%+v", dc.toDeleteCache) +} + func (dc *nameOnlyDirtyCache) getOriginalMembers(_ string) map[string]struct{} { return nil } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index cc42ff45a3..abc943aab1 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -74,8 +74,7 @@ func (iMgr *IPSetManager) Reconcile() { } numRemovedSets := originalNumSets - len(iMgr.setMap) if numRemovedSets > 0 { - // FIXME uncomment this - // klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache to: %+v", numRemovedSets, iMgr.toDeleteCache) + klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache (in Linux, includes original members) to: %+v", numRemovedSets, iMgr.dirtyCache.printDeleteCache()) } } @@ -421,8 +420,10 @@ func (iMgr *IPSetManager) ApplyIPSets() error { return nil } - // FIXME uncomment this - // klog.Infof("[IPSetManager] toAddUpdateCache: %+v \ntoDeleteCache: %+v", iMgr.toAddOrUpdateCache, iMgr.toDeleteCache) + klog.Infof( + "[IPSetManager] dirty caches (in Linux, includes original members). toAddUpdateCache: %s \ntoDeleteCache: %s", + iMgr.dirtyCache.printAddOrUpdateCache(), iMgr.dirtyCache.printDeleteCache(), + ) iMgr.sanitizeDirtyCache() // Call the appropriate apply ipsets From 735178c5f839d8eb9bfe01b5e32d1e1e846211c8 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 6 Apr 2022 10:13:57 -0700 Subject: [PATCH 07/17] some UTs --- .../ipsets/ipsetmanager_linux_test.go | 1451 +++++++++-------- 1 file changed, 733 insertions(+), 718 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go index 74ec3e8aee..2100121e98 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go @@ -1,5 +1,21 @@ package ipsets +import ( + "fmt" + "regexp" + "sort" + "strings" + "testing" + + "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/metrics/promutil" + dptestutils "github.com/Azure/azure-container-networking/npm/pkg/dataplane/testutils" + "github.com/Azure/azure-container-networking/npm/util" + testutils "github.com/Azure/azure-container-networking/test/utils" + "github.com/stretchr/testify/require" +) + const ( saveResult = "create test-list1 list:set size 8\nadd test-list1 test-list2" @@ -15,628 +31,628 @@ azure-npm-777777` var resetIPSetsListOutput = []byte(resetIPSetsListOutputString) +func TestIPSetSave(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + output, err := iMgr.ipsetSave() + require.NoError(t, err) + require.Equal(t, saveResult, string(output)) +} + +func TestIPSetSaveNoMatch(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, ExitCode: 1}, + {Cmd: []string{"grep", "azure-npm-"}}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + output, err := iMgr.ipsetSave() + require.NoError(t, err) + require.Nil(t, output) +} + +// FIXME delete +func TestFileCreator(t *testing.T) { + ioshim := common.NewMockIOShim(nil) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet}) + creator := iMgr.fileCreatorForApply(0, nil) + fmt.Println(creator.ToString()) + + // actualLines := testAndSortRestoreFileString(t, creator.ToString()) + + // expectedLines := []string{ + // fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + // fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + // fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), + // fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), + // fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), + // fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), + // fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), + // fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), + // fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), + // fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), + // fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), + // fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + // fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), + // fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), + // "", + // } + // sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + // dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + // wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + // require.NoError(t, err, "ipset restore should be successful") + // require.False(t, wasFileAltered, "file should not be altered") +} + // TODO test that a reconcile list is updated for all the TestFailure UTs // TODO same exact TestFailure UTs for unknown errors // FIXME uncomment -// func TestApplyIPSets(t *testing.T) { -// type args struct { -// toAddUpdateSets []*IPSetMetadata -// toDeleteSets []*IPSetMetadata -// commandError bool -// } -// tests := []struct { -// name string -// args args -// expectedExecCount int -// wantErr bool -// }{ -// { -// name: "nothing to update", -// args: args{ -// toAddUpdateSets: nil, -// toDeleteSets: nil, -// commandError: false, -// }, -// expectedExecCount: 0, -// wantErr: false, -// }, -// { -// name: "success with just add", -// args: args{ -// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, -// toDeleteSets: nil, -// commandError: false, -// }, -// expectedExecCount: 1, -// wantErr: false, -// }, -// { -// name: "success with just delete", -// args: args{ -// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, -// toDeleteSets: nil, -// commandError: false, -// }, -// expectedExecCount: 1, -// wantErr: false, -// }, -// { -// name: "success with add and delete", -// args: args{ -// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, -// toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, -// commandError: false, -// }, -// expectedExecCount: 1, -// wantErr: false, -// }, -// { -// name: "set is in both delete and add/update cache", -// args: args{ -// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, -// toDeleteSets: []*IPSetMetadata{namespaceSet}, -// commandError: false, -// }, -// expectedExecCount: 1, -// wantErr: false, -// }, -// { -// name: "apply error", -// args: args{ -// toAddUpdateSets: []*IPSetMetadata{namespaceSet}, -// toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, -// commandError: true, -// }, -// expectedExecCount: 1, -// wantErr: true, -// }, -// } -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// metrics.ReinitializeAll() -// calls := GetApplyIPSetsTestCalls(tt.args.toAddUpdateSets, tt.args.toDeleteSets) -// if tt.args.commandError { -// // add an error to the last call (the ipset-restore call) -// // this would potentially cause problems if we used pointers to the TestCmds -// require.Greater(t, len(calls), 0) -// calls[len(calls)-1].ExitCode = 1 -// // then add errors as many times as we retry -// for i := 1; i < maxTryCount; i++ { -// calls = append(calls, testutils.TestCmd{Cmd: ipsetRestoreStringSlice, ExitCode: 1}) -// } -// } -// ioShim := common.NewMockIOShim(calls) -// defer ioShim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) -// iMgr.CreateIPSets(tt.args.toAddUpdateSets) -// for _, set := range tt.args.toDeleteSets { -// iMgr.dirtyCache.delete(NewIPSet(set)) -// } -// err := iMgr.ApplyIPSets() - -// // cache behavior is currently undefined if there's an apply error -// if tt.wantErr { -// require.Error(t, err) -// } else { -// require.NoError(t, err) -// cache := make([]setMembers, 0) -// for _, set := range tt.args.toAddUpdateSets { -// cache = append(cache, setMembers{metadata: set, members: nil}) -// } -// assertExpectedInfo(t, iMgr, &expectedInfo{ -// mainCache: cache, -// toAddUpdateCache: nil, -// toDeleteCache: nil, -// setsForKernel: nil, -// }) -// } - -// execCount, err := metrics.GetIPSetExecCount() -// promutil.NotifyIfErrors(t, err) -// require.Equal(t, tt.expectedExecCount, execCount) -// }) -// } -// } - -// func TestDestroyNPMIPSetsCreatorSuccess(t *testing.T) { -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) -// creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) -// actualLines := strings.Split(creator.ToString(), "\n") -// expectedLines := []string{ -// "-F azure-npm-123456", -// "-F azure-npm-987654", -// "-F azure-npm-777777", -// "-X azure-npm-123456", -// "-X azure-npm-987654", -// "-X azure-npm-777777", -// "", -// } -// dptestutils.AssertEqualLines(t, expectedLines, actualLines) -// require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") -// wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.False(t, wasModified) -// require.NoError(t, err) -// require.Equal(t, 0, *destroyFailureCount, "got unexpected failure count") -// } - -// func TestDestroyNPMIPSetsCreatorErrorHandling(t *testing.T) { -// tests := []struct { -// name string -// call testutils.TestCmd -// expectedLines []string -// expectedFailureCount int -// }{ -// { -// name: "set doesn't exist on flush", -// call: testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 2: The set with the given name does not exist", -// ExitCode: 1, -// }, -// expectedLines: []string{ -// "-F azure-npm-777777", -// "-X azure-npm-123456", -// "-X azure-npm-777777", -// "", -// }, -// expectedFailureCount: 0, -// }, -// { -// name: "some other error on flush", -// call: testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 2: for some other error", -// ExitCode: 1, -// }, -// expectedLines: []string{ -// "-F azure-npm-777777", -// "-X azure-npm-123456", -// "-X azure-npm-777777", -// "", -// }, -// expectedFailureCount: 1, -// }, -// { -// name: "set doesn't exist on destroy", -// call: testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 5: The set with the given name does not exist", -// ExitCode: 1, -// }, -// expectedLines: []string{ -// "-X azure-npm-777777", -// "", -// }, -// expectedFailureCount: 0, -// }, -// { -// name: "some other error on destroy", -// call: testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 5: some other error", -// ExitCode: 1, -// }, -// expectedLines: []string{ -// "-X azure-npm-777777", -// "", -// }, -// expectedFailureCount: 1, -// }, -// } -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// calls := []testutils.TestCmd{tt.call} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) -// creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) -// require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") -// wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.True(t, wasModified) -// require.Error(t, err) -// actualLines := strings.Split(creator.ToString(), "\n") -// dptestutils.AssertEqualLines(t, tt.expectedLines, actualLines) -// require.Equal(t, tt.expectedFailureCount, *destroyFailureCount, "got unexpected failure count") -// }) -// } -// } - -// func TestDestroyNPMIPSets(t *testing.T) { -// tests := []struct { -// name string -// calls []testutils.TestCmd -// wantErr bool -// }{ -// { -// name: "success with no results from grep", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, ExitCode: 1}, -// }, -// wantErr: false, -// }, -// { -// name: "successfully delete sets", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// fakeRestoreSuccessCommand, -// }, -// wantErr: false, -// }, -// { -// name: "grep error", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, HasStartError: true, PipedToCommand: true, ExitCode: 1}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// }, -// wantErr: true, -// }, -// { -// name: "restore error from max tries", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// }, -// wantErr: true, -// }, -// { -// name: "successfully restore, but fail to flush/destroy 1 set since the set doesn't exist when flushing", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// { -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 2: The set with the given name does not exist", -// ExitCode: 1, -// }, -// fakeRestoreSuccessCommand, -// }, -// wantErr: false, -// }, -// { -// name: "successfully restore, but fail to flush/destroy 1 set due to other flush error", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// { -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 2: for some other error", -// ExitCode: 1, -// }, -// fakeRestoreSuccessCommand, -// }, -// wantErr: false, -// }, -// { -// name: "successfully restore, but fail to destroy 1 set since the set doesn't exist when destroying", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// { -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 5: The set with the given name does not exist", -// ExitCode: 1, -// }, -// fakeRestoreSuccessCommand, -// }, -// wantErr: false, -// }, -// { -// name: "successfully restore, but fail to destroy 1 set due to other destroy error", -// calls: []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, -// { -// Cmd: ipsetRestoreStringSlice, -// Stdout: "Error in line 5: for some other error", -// ExitCode: 1, -// }, -// fakeRestoreSuccessCommand, -// }, -// wantErr: false, -// }, -// } - -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// ioshim := common.NewMockIOShim(tt.calls) -// defer ioshim.VerifyCalls(t, tt.calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) -// err := iMgr.resetIPSets() -// if tt.wantErr { -// require.Error(t, err) -// } else { -// require.NoError(t, err) -// } -// }) -// } -// } - -// // identical to TestResetIPSets in ipsetmanager_test.go except an error occurs -// // makes sure that the cache and metrics are reset despite error -// func TestResetIPSetsOnFailure(t *testing.T) { -// metrics.ReinitializeAll() -// calls := []testutils.TestCmd{ -// {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true, HasStartError: true}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// } -// ioShim := common.NewMockIOShim(calls) -// defer ioShim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) - -// iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet, keyLabelOfPodSet}) - -// metrics.IncNumIPSets() -// metrics.IncNumIPSets() -// metrics.AddEntryToIPSet("test1") -// metrics.AddEntryToIPSet("test1") -// metrics.AddEntryToIPSet("test2") - -// require.NoError(t, iMgr.ResetIPSets()) - -// assertExpectedInfo(t, iMgr, &expectedInfo{ -// mainCache: nil, -// toAddUpdateCache: nil, -// toDeleteCache: nil, -// setsForKernel: nil, -// }) -// } - -// func TestApplyIPSetsSuccessWithoutSave(t *testing.T) { -// // no sets to add/update, so don't call ipset save -// calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // delete a set so the file isn't empty (otherwise the creator won't even call the exec command) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestNSSet.PrefixName, util.SoftDelete) -// err := iMgr.applyIPSets() -// require.NoError(t, err) -// } - -// func TestApplyIPSetsSuccessWithSave(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// {Cmd: ipsetRestoreStringSlice}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // create a set so we run ipset save -// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) -// err := iMgr.applyIPSets() -// require.NoError(t, err) -// } - -// func TestApplyIPSetsFailureOnSave(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, HasStartError: true, PipedToCommand: true, ExitCode: 1}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // create a set so we run ipset save -// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) -// err := iMgr.applyIPSets() -// require.Error(t, err) -// } - -// func TestApplyIPSetsFailureOnRestore(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// // fail 3 times because this is our max try count -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // create a set so we run ipset save -// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) -// err := iMgr.applyIPSets() -// require.Error(t, err) -// } - -// func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, -// {Cmd: ipsetRestoreStringSlice}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // create a set so we run ipset save -// iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) -// err := iMgr.applyIPSets() -// require.NoError(t, err) -// } - -// func TestIPSetSave(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, -// {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// output, err := iMgr.ipsetSave() -// require.NoError(t, err) -// require.Equal(t, saveResult, string(output)) -// } - -// func TestIPSetSaveNoMatch(t *testing.T) { -// calls := []testutils.TestCmd{ -// {Cmd: ipsetSaveStringSlice, ExitCode: 1}, -// {Cmd: []string{"grep", "azure-npm-"}}, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// output, err := iMgr.ipsetSave() -// require.NoError(t, err) -// require.Nil(t, output) -// } - -// func TestCreateForAllSetTypes(t *testing.T) { -// // without save file -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) -// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) - -// creator := iMgr.fileCreatorForApply(len(calls), nil) -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) - -// expectedLines := []string{ -// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), -// fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), -// fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), -// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), -// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), -// fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), -// "", -// } -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err, "ipset restore should be successful") -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestDestroy(t *testing.T) { -// // without save file -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // remove some members and destroy some sets -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) -// require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) -// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) -// require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), nil) -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) - -// expectedLines := []string{ -// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), -// fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), -// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), -// fmt.Sprintf("-F %s", TestCIDRSet.HashedName), -// fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), -// fmt.Sprintf("-X %s", TestCIDRSet.HashedName), -// fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), -// "", -// } -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err, "ipset restore should be successful") -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnCreateForNewSet(t *testing.T) { -// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order -// // test logic: -// // - delete a set -// // - create three sets, each with two members. the second set to appear will fail to be created -// errorLineNum := 2 -// setToCreateAlreadyExistsCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// // add all of these members to the kernel -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member -// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) - -// // get original creator and run it the first time -// creator := iMgr.fileCreatorForApply(len(calls), nil) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated -// removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// originalLength := len(expectedLines) -// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") -// require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } +func TestApplyIPSets(t *testing.T) { + type args struct { + toAddUpdateSets []*IPSetMetadata + toDeleteSets []*IPSetMetadata + commandError bool + } + tests := []struct { + name string + args args + expectedExecCount int + wantErr bool + }{ + { + name: "nothing to update", + args: args{ + toAddUpdateSets: nil, + toDeleteSets: nil, + commandError: false, + }, + expectedExecCount: 0, + wantErr: false, + }, + { + name: "success with just add", + args: args{ + toAddUpdateSets: []*IPSetMetadata{namespaceSet}, + toDeleteSets: nil, + commandError: false, + }, + expectedExecCount: 1, + wantErr: false, + }, + { + name: "success with just delete", + args: args{ + toAddUpdateSets: []*IPSetMetadata{namespaceSet}, + toDeleteSets: nil, + commandError: false, + }, + expectedExecCount: 1, + wantErr: false, + }, + { + name: "success with add and delete", + args: args{ + toAddUpdateSets: []*IPSetMetadata{namespaceSet}, + toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, + commandError: false, + }, + expectedExecCount: 1, + wantErr: false, + }, + { + name: "set is in both delete and add/update cache", + args: args{ + toAddUpdateSets: []*IPSetMetadata{namespaceSet}, + toDeleteSets: []*IPSetMetadata{namespaceSet}, + commandError: false, + }, + expectedExecCount: 1, + wantErr: false, + }, + { + name: "apply error", + args: args{ + toAddUpdateSets: []*IPSetMetadata{namespaceSet}, + toDeleteSets: []*IPSetMetadata{keyLabelOfPodSet}, + commandError: true, + }, + expectedExecCount: 1, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + metrics.ReinitializeAll() + calls := GetApplyIPSetsTestCalls(tt.args.toAddUpdateSets, tt.args.toDeleteSets) + if tt.args.commandError { + // add an error to the last call (the ipset-restore call) + // this would potentially cause problems if we used pointers to the TestCmds + require.Greater(t, len(calls), 0) + calls[len(calls)-1].ExitCode = 1 + // then add errors as many times as we retry + for i := 1; i < maxTryCount; i++ { + calls = append(calls, testutils.TestCmd{Cmd: ipsetRestoreStringSlice, ExitCode: 1}) + } + } + ioShim := common.NewMockIOShim(calls) + defer ioShim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) + iMgr.CreateIPSets(tt.args.toAddUpdateSets) + for _, set := range tt.args.toDeleteSets { + iMgr.dirtyCache.delete(NewIPSet(set)) + } + err := iMgr.ApplyIPSets() + + // cache behavior is currently undefined if there's an apply error + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + cache := make([]setMembers, 0) + for _, set := range tt.args.toAddUpdateSets { + cache = append(cache, setMembers{metadata: set, members: nil}) + } + assertExpectedInfo(t, iMgr, &expectedInfo{ + mainCache: cache, + toAddUpdateCache: nil, + toDeleteCache: nil, + setsForKernel: nil, + }) + } + + execCount, err := metrics.GetIPSetExecCount() + promutil.NotifyIfErrors(t, err) + require.Equal(t, tt.expectedExecCount, execCount) + }) + } +} + +func TestDestroyNPMIPSetsCreatorSuccess(t *testing.T) { + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) + actualLines := strings.Split(creator.ToString(), "\n") + expectedLines := []string{ + "-F azure-npm-123456", + "-F azure-npm-987654", + "-F azure-npm-777777", + "-X azure-npm-123456", + "-X azure-npm-987654", + "-X azure-npm-777777", + "", + } + dptestutils.AssertEqualLines(t, expectedLines, actualLines) + require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") + wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.False(t, wasModified) + require.NoError(t, err) + require.Equal(t, 0, *destroyFailureCount, "got unexpected failure count") +} + +func TestDestroyNPMIPSetsCreatorErrorHandling(t *testing.T) { + tests := []struct { + name string + call testutils.TestCmd + expectedLines []string + expectedFailureCount int + }{ + { + name: "set doesn't exist on flush", + call: testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 2: The set with the given name does not exist", + ExitCode: 1, + }, + expectedLines: []string{ + "-F azure-npm-777777", + "-X azure-npm-123456", + "-X azure-npm-777777", + "", + }, + expectedFailureCount: 0, + }, + { + name: "some other error on flush", + call: testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 2: for some other error", + ExitCode: 1, + }, + expectedLines: []string{ + "-F azure-npm-777777", + "-X azure-npm-123456", + "-X azure-npm-777777", + "", + }, + expectedFailureCount: 1, + }, + { + name: "set doesn't exist on destroy", + call: testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 5: The set with the given name does not exist", + ExitCode: 1, + }, + expectedLines: []string{ + "-X azure-npm-777777", + "", + }, + expectedFailureCount: 0, + }, + { + name: "some other error on destroy", + call: testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 5: some other error", + ExitCode: 1, + }, + expectedLines: []string{ + "-X azure-npm-777777", + "", + }, + expectedFailureCount: 1, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + calls := []testutils.TestCmd{tt.call} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + creator, numSets, destroyFailureCount := iMgr.fileCreatorForReset(resetIPSetsListOutput) + require.Equal(t, resetIPSetsNumGreppedSets, numSets, "got unexpected num sets") + wasModified, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.True(t, wasModified) + require.Error(t, err) + actualLines := strings.Split(creator.ToString(), "\n") + dptestutils.AssertEqualLines(t, tt.expectedLines, actualLines) + require.Equal(t, tt.expectedFailureCount, *destroyFailureCount, "got unexpected failure count") + }) + } +} + +func TestDestroyNPMIPSets(t *testing.T) { + tests := []struct { + name string + calls []testutils.TestCmd + wantErr bool + }{ + { + name: "success with no results from grep", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, ExitCode: 1}, + }, + wantErr: false, + }, + { + name: "successfully delete sets", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + fakeRestoreSuccessCommand, + }, + wantErr: false, + }, + { + name: "grep error", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, HasStartError: true, PipedToCommand: true, ExitCode: 1}, + {Cmd: []string{"grep", "azure-npm-"}}, + }, + wantErr: true, + }, + { + name: "restore error from max tries", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + }, + wantErr: true, + }, + { + name: "successfully restore, but fail to flush/destroy 1 set since the set doesn't exist when flushing", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + { + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 2: The set with the given name does not exist", + ExitCode: 1, + }, + fakeRestoreSuccessCommand, + }, + wantErr: false, + }, + { + name: "successfully restore, but fail to flush/destroy 1 set due to other flush error", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + { + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 2: for some other error", + ExitCode: 1, + }, + fakeRestoreSuccessCommand, + }, + wantErr: false, + }, + { + name: "successfully restore, but fail to destroy 1 set since the set doesn't exist when destroying", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + { + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 5: The set with the given name does not exist", + ExitCode: 1, + }, + fakeRestoreSuccessCommand, + }, + wantErr: false, + }, + { + name: "successfully restore, but fail to destroy 1 set due to other destroy error", + calls: []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: resetIPSetsListOutputString}, + { + Cmd: ipsetRestoreStringSlice, + Stdout: "Error in line 5: for some other error", + ExitCode: 1, + }, + fakeRestoreSuccessCommand, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ioshim := common.NewMockIOShim(tt.calls) + defer ioshim.VerifyCalls(t, tt.calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + err := iMgr.resetIPSets() + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +// identical to TestResetIPSets in ipsetmanager_test.go except an error occurs +// makes sure that the cache and metrics are reset despite error +func TestResetIPSetsOnFailure(t *testing.T) { + metrics.ReinitializeAll() + calls := []testutils.TestCmd{ + {Cmd: []string{"ipset", "list", "--name"}, PipedToCommand: true, HasStartError: true}, + {Cmd: []string{"grep", "azure-npm-"}}, + } + ioShim := common.NewMockIOShim(calls) + defer ioShim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) + + iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet, keyLabelOfPodSet}) + + metrics.IncNumIPSets() + metrics.IncNumIPSets() + metrics.AddEntryToIPSet("test1") + metrics.AddEntryToIPSet("test1") + metrics.AddEntryToIPSet("test2") + + require.NoError(t, iMgr.ResetIPSets()) + + assertExpectedInfo(t, iMgr, &expectedInfo{ + mainCache: nil, + toAddUpdateCache: nil, + toDeleteCache: nil, + setsForKernel: nil, + }) +} + +func TestApplyIPSetsSuccess(t *testing.T) { + // no sets to add/update, so don't call ipset save + calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // delete a set so the file isn't empty (otherwise the creator won't even call the exec command) + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestNSSet.PrefixName, util.SoftDelete) + err := iMgr.applyIPSets() + require.NoError(t, err) +} + +func TestApplyIPSetsFailureOnRestore(t *testing.T) { + calls := []testutils.TestCmd{ + // fail 3 times because this is our max try count + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + } + ioshim := common.NewMockIOShim(calls) + // defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err := iMgr.applyIPSets() + require.Error(t, err) +} + +func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err := iMgr.applyIPSets() + require.NoError(t, err) +} + +func TestCreateForAllSetTypes(t *testing.T) { + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) + iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) + + creator := iMgr.fileCreatorForApply(len(calls), nil) + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + + expectedLines := []string{ + fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), + fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), + fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), + fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), + fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), + "", + } + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestDestroy(t *testing.T) { + // without save file + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // remove some members and destroy some sets + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) + require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) + + creator := iMgr.fileCreatorForApply(len(calls), nil) + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + + expectedLines := []string{ + fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), + fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + fmt.Sprintf("-F %s", TestCIDRSet.HashedName), + fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), + fmt.Sprintf("-X %s", TestCIDRSet.HashedName), + fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), + "", + } + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestFailureOnCreateForNewSet(t *testing.T) { + // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order + // test logic: + // - delete a set + // - create three sets, each with two members. the second set to appear will fail to be created + errorLineNum := 2 + setToCreateAlreadyExistsCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), + ExitCode: 1, + } + calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // add all of these members to the kernel + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member + iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) + + // get original creator and run it the first time + creator := iMgr.fileCreatorForApply(len(calls), nil) + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated + removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run + originalLength := len(expectedLines) + expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") + require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") +} // func TestFailureOnCreateForSetInKernel(t *testing.T) { // // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order @@ -944,107 +960,106 @@ var resetIPSetsListOutput = []byte(resetIPSetsListOutputString) // require.False(t, wasFileAltered, "file should not be altered") // } -// func testAndSortRestoreFileString(t *testing.T, multilineString string) []string { -// return testAndSortRestoreFileLines(t, strings.Split(multilineString, "\n")) -// } - -// // make sure file goes in order of creates, adds/deletes, flushes, then destroys -// // then sort those sections and return the lines in an array -// func testAndSortRestoreFileLines(t *testing.T, lines []string) []string { -// if len(lines) == 0 { -// return lines -// } -// require.True(t, lines[len(lines)-1] == "", "restore file must end with blank line") -// lines = lines[:len(lines)-1] // remove the blank line - -// // order of operation groups in restore file (can have groups with multiple possible operatoins) -// operationGroups := [][]string{ -// {"-N"}, // creates -// {"-A", "-D"}, // adds/deletes -// {"-F"}, // flushes -// {"-X"}, // destroys -// } -// result := make([]string, 0, len(lines)) -// groupIndex := 0 -// groupStartIndex := 0 -// k := 0 -// for k < len(lines) { -// for k < len(lines) { -// // iterate until we reach an operation not in the current operation group -// operation := lines[k][0:2] -// expectedOperations := operationGroups[groupIndex] -// if !isStringInSlice(operation, expectedOperations) { -// require.True(t, groupIndex < len(operationGroups)-1, "ran out of operation groups. got operation %s", operation) -// operationLines := lines[groupStartIndex:k] -// sort.Strings(operationLines) -// result = append(result, operationLines...) -// groupStartIndex = k -// groupIndex++ -// break -// } -// k++ -// } -// } -// // add the remaining lines since the final operation group won't pass through the if statement in the loop above -// operatrionLines := lines[groupStartIndex:] -// sort.Strings(operatrionLines) -// result = append(result, operatrionLines...) -// result = append(result, "") // add the blank line -// return result -// } - -// func hashedNameOfSetImpacted(t *testing.T, operation string, lines []string, lineNum int) string { -// lineNumIndex := lineNum - 1 -// line := lines[lineNumIndex] -// pattern := fmt.Sprintf(`\%s (azure-npm-\d+)`, operation) -// re := regexp.MustCompile(pattern) -// results := re.FindStringSubmatch(line) -// require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) -// return results[1] // second item in slice is the group surrounded by () -// } - -// func memberNameOfSetImpacted(t *testing.T, lines []string, lineNum int) string { -// lineNumIndex := lineNum - 1 -// line := lines[lineNumIndex] -// pattern := `\-[AD] azure-npm-\d+ (.*)` -// re := regexp.MustCompile(pattern) -// member := re.FindStringSubmatch(line)[1] -// results := re.FindStringSubmatch(line) -// require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) -// return member -// } - -// func isStringInSlice(item string, values []string) bool { -// success := false -// for _, value := range values { -// if item == value { -// success = true -// break -// } -// } -// return success -// } - -// func requireStringInSlice(t *testing.T, item string, values []string) { -// require.Truef(t, isStringInSlice(item, values), "item %s was not one of the possible values %+v", item, values) -// } - -// // remove lines that start with the operation (include the dash in the operations) e.g. -// // -A 1.2.3.4 -// // -D 1.2.3.4 -// // -X -// func removeOperationsForSet(lines []string, hashedSetName, operation string) []string { -// operationRegex := regexp.MustCompile(fmt.Sprintf(`\%s %s`, operation, hashedSetName)) -// goodLines := []string{} -// for _, line := range lines { -// if !operationRegex.MatchString(line) { -// goodLines = append(goodLines, line) -// } -// } -// return goodLines -// } +func testAndSortRestoreFileString(t *testing.T, multilineString string) []string { + return testAndSortRestoreFileLines(t, strings.Split(multilineString, "\n")) +} + +// make sure file goes in order of creates, adds/deletes, flushes, then destroys +// then sort those sections and return the lines in an array +func testAndSortRestoreFileLines(t *testing.T, lines []string) []string { + if len(lines) == 0 { + return lines + } + require.True(t, lines[len(lines)-1] == "", "restore file must end with blank line") + lines = lines[:len(lines)-1] // remove the blank line + + // order of operation groups in restore file (can have groups with multiple possible operatoins) + operationGroups := [][]string{ + {"-N"}, // creates + {"-A", "-D"}, // adds/deletes + {"-F"}, // flushes + {"-X"}, // destroys + } + result := make([]string, 0, len(lines)) + groupIndex := 0 + groupStartIndex := 0 + k := 0 + for k < len(lines) { + for k < len(lines) { + // iterate until we reach an operation not in the current operation group + operation := lines[k][0:2] + expectedOperations := operationGroups[groupIndex] + if !isStringInSlice(operation, expectedOperations) { + require.True(t, groupIndex < len(operationGroups)-1, "ran out of operation groups. got operation %s", operation) + operationLines := lines[groupStartIndex:k] + sort.Strings(operationLines) + result = append(result, operationLines...) + groupStartIndex = k + groupIndex++ + break + } + k++ + } + } + // add the remaining lines since the final operation group won't pass through the if statement in the loop above + operatrionLines := lines[groupStartIndex:] + sort.Strings(operatrionLines) + result = append(result, operatrionLines...) + result = append(result, "") // add the blank line + return result +} + +func hashedNameOfSetImpacted(t *testing.T, operation string, lines []string, lineNum int) string { + lineNumIndex := lineNum - 1 + line := lines[lineNumIndex] + pattern := fmt.Sprintf(`\%s (azure-npm-\d+)`, operation) + re := regexp.MustCompile(pattern) + results := re.FindStringSubmatch(line) + require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) + return results[1] // second item in slice is the group surrounded by () +} + +func memberNameOfSetImpacted(t *testing.T, lines []string, lineNum int) string { + lineNumIndex := lineNum - 1 + line := lines[lineNumIndex] + pattern := `\-[AD] azure-npm-\d+ (.*)` + re := regexp.MustCompile(pattern) + member := re.FindStringSubmatch(line)[1] + results := re.FindStringSubmatch(line) + require.Equal(t, 2, len(results), "expected to find a match with regex pattern %s for line: %s", pattern, line) + return member +} + +func isStringInSlice(item string, values []string) bool { + success := false + for _, value := range values { + if item == value { + success = true + break + } + } + return success +} + +func requireStringInSlice(t *testing.T, item string, values []string) { + require.Truef(t, isStringInSlice(item, values), "item %s was not one of the possible values %+v", item, values) +} + +// remove lines that start with the operation (include the dash in the operations) e.g. +// -A 1.2.3.4 +// -D 1.2.3.4 +// -X +func removeOperationsForSet(lines []string, hashedSetName, operation string) []string { + operationRegex := regexp.MustCompile(fmt.Sprintf(`\%s %s`, operation, hashedSetName)) + goodLines := []string{} + for _, line := range lines { + if !operationRegex.MatchString(line) { + goodLines = append(goodLines, line) + } + } + return goodLines +} -// FIXME LEAVE COMMENTED FOR LATER // func TestNextCreateLine(t *testing.T) { // createLine := "create test-list1 list:set size 8" // addLine := "add test-set1 1.2.3.4" From a0b6cc04e278d6d683f51abbea37733666a15e80 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 6 Apr 2022 10:14:25 -0700 Subject: [PATCH 08/17] dirty cache UTs --- npm/pkg/dataplane/ipsets/dirtycache_test.go | 194 ++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 npm/pkg/dataplane/ipsets/dirtycache_test.go diff --git a/npm/pkg/dataplane/ipsets/dirtycache_test.go b/npm/pkg/dataplane/ipsets/dirtycache_test.go new file mode 100644 index 0000000000..f8773a56ba --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache_test.go @@ -0,0 +1,194 @@ +package ipsets + +import ( + "testing" + + "github.com/Azure/azure-container-networking/npm/util" + "github.com/stretchr/testify/require" +) + +// members are only important for Linux +type dirtyCacheResults struct { + // map of prefixed name to members + toAddOrUpdate map[string][]string + // map of prefixed name to members + toDelete map[string][]string +} + +const ( + ip = "1.2.3.4" + podKey = "pod1" +) + +func TestDirtyCacheReset(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + set2 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.create(set1) + dc.delete(set2) + dc.reset() + assertDirtyCache(t, dc, &dirtyCacheResults{}) +} + +func TestDirtyCacheCreate(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + set2 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + dc := newDirtyCache() + dc.create(set1) + // members are ignored on create + set2.IPPodKey[ip] = podKey + dc.create(set2) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {}, + set2.Name: {}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheCreateAfterDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.delete(set1) + // original members shouldn't get updated + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {ip}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheUpdateNew(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.update(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {ip}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheUpdateOld(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + // original members shouldn't get updated + set1.IPPodKey[ip] = podKey + dc.update(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheUpdateTwice(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.update(set1) + // original members shouldn't get updated + set1.IPPodKey[ip] = podKey + dc.update(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheUpdateAfterDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.delete(set1) + // original members shouldn't get updated + set1.IPPodKey[ip] = podKey + dc.update(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: map[string][]string{ + set1.Name: {}, + }, + toDelete: nil, + }) +} + +func TestDirtyCacheDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.delete(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: nil, + toDelete: map[string][]string{ + set1.Name: {ip}, + }, + }) +} + +func TestDirtyCacheDeleteOld(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.update(set1) + // original members shouldn't get updated + delete(set1.IPPodKey, ip) + dc.delete(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: nil, + toDelete: map[string][]string{ + set1.Name: {ip}, + }, + }) +} + +func TestDirtyCacheDeleteTwice(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip] = podKey + dc.update(set1) + // original members shouldn't get updated + delete(set1.IPPodKey, ip) + dc.delete(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toAddOrUpdate: nil, + toDelete: map[string][]string{ + set1.Name: {ip}, + }, + }) +} + +func assertDirtyCache(t *testing.T, dc dirtyCacheMaintainer, expected *dirtyCacheResults) { + require.Equal(t, len(expected.toAddOrUpdate), dc.numSetsToAddOrUpdate(), "unexpected number of sets to add or update") + require.Equal(t, len(expected.toDelete), dc.numSetsToDelete(), "unexpected number of sets to delete") + for setName, members := range expected.toAddOrUpdate { + require.True(t, dc.isSetToAddOrUpdate(setName), "set %s should be added/updated", setName) + require.False(t, dc.isSetToDelete(setName), "set %s should not be deleted", setName) + if !util.IsWindowsDP() { + require.Equal(t, stringSliceToSet(members), dc.getOriginalMembers(setName), "unexpected original members for set %s", setName) + } + } + for setName, members := range expected.toDelete { + require.True(t, dc.isSetToDelete(setName), "set %s should be deleted", setName) + require.False(t, dc.isSetToAddOrUpdate(setName), "set %s should not be added/updated", setName) + if !util.IsWindowsDP() { + require.Equal(t, stringSliceToSet(members), dc.getOriginalMembers(setName), "unexpected original members for set %s", setName) + } + } +} + +func stringSliceToSet(s []string) map[string]struct{} { + m := make(map[string]struct{}, len(s)) + for _, v := range s { + m[v] = struct{}{} + } + return m +} From 2b3a6ecc600b4c768b49cd88c09a6c4cf1b3daac Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 6 Apr 2022 15:52:08 -0700 Subject: [PATCH 09/17] fix windows build problem --- npm/pkg/dataplane/ipsets/dirtycache_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 9e646d8abf..0cf32b7adc 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -1,5 +1,7 @@ package ipsets +import "fmt" + type nameOnlyDirtyCache struct { toAddOrUpdateCache map[string]struct{} toDeleteCache map[string]struct{} @@ -31,7 +33,6 @@ func (dc *nameOnlyDirtyCache) delete(originalSet *IPSet) { func putIntoAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struct{}) { if _, ok := intoCache[set.Name]; ok { - // NOTE: could throw error if setType is different return } intoCache[set.Name] = struct{}{} From bab47ba4e4ac63b129606e3c00b277aeb7ed7172 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 11 Apr 2022 10:42:48 -0700 Subject: [PATCH 10/17] dirty cache with members to add/delete & keep old apply with save file code --- npm/pkg/dataplane/ipsets/dirtycache.go | 36 +- npm/pkg/dataplane/ipsets/dirtycache_linux.go | 259 ++- npm/pkg/dataplane/ipsets/dirtycache_test.go | 294 +-- .../dataplane/ipsets/dirtycache_windows.go | 50 +- npm/pkg/dataplane/ipsets/ipsetmanager.go | 35 +- .../dataplane/ipsets/ipsetmanager_linux.go | 474 +++-- .../ipsets/ipsetmanager_linux_test.go | 1743 +++++++++-------- .../dataplane/ipsets/ipsetmanager_windows.go | 2 +- 8 files changed, 1595 insertions(+), 1298 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index 5f806e3a22..5ffc1c9dcc 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -1,19 +1,28 @@ package ipsets -// dirtyCacheMaintainer will maintain the dirty cache -type dirtyCacheMaintainer interface { +/* + dirtyCacheInterface will maintain the dirty cache. + It may maintain membersToAdd and membersToDelete. + Members are either IPs, CIDRs, IP-Port pairs, or prefixed set names if the parent is a list. + + Assumptions: + FIXME: TODO +*/ +type dirtyCacheInterface interface { // reset empties dirty cache reset() // create will mark the new set to be created. - create(newSet *IPSet) - // update will mark the set to be updated and may note the original members of the set. - update(originalSet *IPSet) + create(set *IPSet) + // addMember will mark the set to be updated and track the member to be added (if implemented). + addMember(set *IPSet, member string) + // deleteMember will mark the set to be updated and track the member to be deleted (if implemented). + deleteMember(set *IPSet, member string) // delete will mark the set to be deleted in the cache - delete(originalSet *IPSet) - // getSetsToAddOrUpdate returns the list of set names to be added or updated - getSetsToAddOrUpdate() []string - // getSetsToDelete returns the list of set names to be deleted - getSetsToDelete() []string + destroy(set *IPSet) + // getSetsToAddOrUpdate returns the set names to be added or updated + getSetsToAddOrUpdate() map[string]struct{} + // getSetsToDelete returns the set names to be deleted + getSetsToDelete() map[string]struct{} // numSetsToAddOrUpdate returns the number of sets to be added or updated numSetsToAddOrUpdate() int // numSetsToDelete returns the number of sets to be deleted @@ -26,7 +35,8 @@ type dirtyCacheMaintainer interface { printAddOrUpdateCache() string // printDeleteCache returns a string representation of the delete cache printDeleteCache() string - // getOriginalMembers returns the original members of the set before it was dirty. - // members are either IPs, CIDRs, IP-Port pairs, or prefixed set names if the parent is a list - getOriginalMembers(setName string) map[string]struct{} + // getOriginalMembers returns the members which should be added for the set. + getMembersToAdd(setName string) map[string]struct{} + // getOriginalMembers returns the members which should be deleted for the set. + getMembersToDelete(setName string) map[string]struct{} } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index ba7cd02a7d..1599f6ca26 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -1,119 +1,234 @@ package ipsets -import "fmt" +import ( + "fmt" -type memberAwareDirtyCache struct { - // map of prefixed set names to original members - toAddOrUpdateCache map[string]map[string]struct{} - // map of prefixed set names to original members - toDeleteCache map[string]map[string]struct{} + "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/util" + "k8s.io/klog" +) + +type dirtyCache struct { + // all maps have keys of set names and values of members to add/delete + toCreateCache map[string]*memberDiff + toUpdateCache map[string]*memberDiff + toDestroyCache map[string]*memberDiff +} + +type memberDiff struct { + membersToAdd map[string]struct{} + membersToDelete map[string]struct{} } -func newDirtyCache() dirtyCacheMaintainer { - return &memberAwareDirtyCache{ - toAddOrUpdateCache: make(map[string]map[string]struct{}), - toDeleteCache: make(map[string]map[string]struct{}), +func newMemberDiff() *memberDiff { + return &memberDiff{ + membersToAdd: make(map[string]struct{}), + membersToDelete: make(map[string]struct{}), } } -func (dc *memberAwareDirtyCache) reset() { - dc.toAddOrUpdateCache = make(map[string]map[string]struct{}) - dc.toDeleteCache = make(map[string]map[string]struct{}) +func newDirtyCache() dirtyCacheInterface { + dc := &dirtyCache{} + dc.reset() + return dc +} + +func (dc *dirtyCache) reset() { + dc.toCreateCache = make(map[string]*memberDiff) + dc.toUpdateCache = make(map[string]*memberDiff) + dc.toDestroyCache = make(map[string]*memberDiff) } -func (dc *memberAwareDirtyCache) create(newSet *IPSet) { - setName := newSet.Name - if _, ok := dc.toAddOrUpdateCache[setName]; ok { - return +func (dc *dirtyCache) create(set *IPSet) { + // error checking + if _, ok := dc.toCreateCache[set.Name]; ok { + msg := fmt.Sprintf("create: set %s should not already be in the toCreateCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) } - info, ok := dc.toDeleteCache[setName] - if !ok { - info = make(map[string]struct{}) + if _, ok := dc.toUpdateCache[set.Name]; ok { + msg := fmt.Sprintf("create: set %s should not be in the toUpdateCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) } - dc.toAddOrUpdateCache[setName] = info - delete(dc.toDeleteCache, setName) -} -func (dc *memberAwareDirtyCache) update(originalSet *IPSet) { - putIntoAndRemoveFromOther(originalSet, dc.toAddOrUpdateCache, dc.toDeleteCache) + diff, ok := dc.toDestroyCache[set.Name] + if ok { + // transfer from toDestroyCache to toUpdateCache and maintain member diff + dc.toUpdateCache[set.Name] = diff + delete(dc.toDestroyCache, set.Name) + } else { + // put in the toCreateCache and mark all current members as membersToAdd + var members map[string]struct{} + if set.Kind == HashSet { + members = make(map[string]struct{}, len(set.IPPodKey)) + for ip := range set.IPPodKey { + members[ip] = struct{}{} + } + } else { + members = make(map[string]struct{}, len(set.MemberIPSets)) + for _, memberSet := range set.MemberIPSets { + members[memberSet.HashedName] = struct{}{} + } + } + dc.toCreateCache[set.Name] = &memberDiff{ + membersToAdd: members, + } + } + fmt.Println("here") } -func (dc *memberAwareDirtyCache) delete(originalSet *IPSet) { - putIntoAndRemoveFromOther(originalSet, dc.toDeleteCache, dc.toAddOrUpdateCache) +func (dc *dirtyCache) addMember(set *IPSet, member string) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("addMember: set %s should not be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + } + + diff := dc.getCreateOrUpdateDiff(set) + _, ok := diff.membersToDelete[member] + if ok { + delete(diff.membersToDelete, member) + } else { + diff.membersToAdd[member] = struct{}{} + } } -func putIntoAndRemoveFromOther(originalSet *IPSet, intoCache, fromCache map[string]map[string]struct{}) { - setName := originalSet.Name - if _, ok := intoCache[setName]; ok { - return +func (dc *dirtyCache) deleteMember(set *IPSet, member string) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("deleteMember: set %s should not be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + } + + diff := dc.getCreateOrUpdateDiff(set) + _, ok := diff.membersToAdd[member] + if ok { + delete(diff.membersToAdd, member) + } else { + diff.membersToDelete[member] = struct{}{} } - members, ok := fromCache[setName] +} + +func (dc *dirtyCache) getCreateOrUpdateDiff(set *IPSet) *memberDiff { + diff, ok := dc.toCreateCache[set.Name] if !ok { - setType := originalSet.Type - members = make(map[string]struct{}) - if setType.getSetKind() == HashSet { - for member := range originalSet.IPPodKey { - members[member] = struct{}{} + diff, ok = dc.toUpdateCache[set.Name] + if !ok { + diff = newMemberDiff() + dc.toUpdateCache[set.Name] = diff + } + } + return diff +} + +func (dc *dirtyCache) destroy(set *IPSet) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("destroy: set %s should not already be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + } + + if _, ok := dc.toCreateCache[set.Name]; !ok { + // modify the diff in the toUpdateCache + // mark all current members as membersToDelete to accommodate force delete + if set.Kind == HashSet { + for ip := range set.IPPodKey { + dc.deleteMember(set, ip) } } else { - for memberName := range originalSet.MemberIPSets { - members[memberName] = struct{}{} + for _, memberSet := range set.MemberIPSets { + dc.deleteMember(set, memberSet.HashedName) } } } - intoCache[setName] = members - delete(fromCache, setName) + // put the diff in the toDestroyCache + diff := dc.getCreateOrUpdateDiff(set) + dc.toDestroyCache[set.Name] = diff + delete(dc.toCreateCache, set.Name) + delete(dc.toUpdateCache, set.Name) } -func (dc *memberAwareDirtyCache) getSetsToAddOrUpdate() []string { - setsToAddOrUpdate := make([]string, 0, len(dc.toAddOrUpdateCache)) - for setName := range dc.toAddOrUpdateCache { - setsToAddOrUpdate = append(setsToAddOrUpdate, setName) +func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { + sets := make(map[string]struct{}, len(dc.toCreateCache)+len(dc.toUpdateCache)) + for set := range dc.toCreateCache { + sets[set] = struct{}{} + } + for set := range dc.toUpdateCache { + sets[set] = struct{}{} } - return setsToAddOrUpdate + return sets } -func (dc *memberAwareDirtyCache) getSetsToDelete() []string { - setsToDelete := make([]string, 0, len(dc.toDeleteCache)) - for setName := range dc.toDeleteCache { - setsToDelete = append(setsToDelete, setName) +func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { + sets := make(map[string]struct{}, len(dc.toDestroyCache)) + for setName := range dc.toDestroyCache { + sets[setName] = struct{}{} } - return setsToDelete + return sets } -func (dc *memberAwareDirtyCache) numSetsToAddOrUpdate() int { - return len(dc.toAddOrUpdateCache) +func (dc *dirtyCache) numSetsToAddOrUpdate() int { + return len(dc.toCreateCache) + len(dc.toUpdateCache) } -func (dc *memberAwareDirtyCache) numSetsToDelete() int { - return len(dc.toDeleteCache) +func (dc *dirtyCache) numSetsToDelete() int { + return len(dc.toDestroyCache) } -func (dc *memberAwareDirtyCache) isSetToAddOrUpdate(setName string) bool { - _, ok := dc.toAddOrUpdateCache[setName] - return ok +func (dc *dirtyCache) isSetToAddOrUpdate(setName string) bool { + _, ok1 := dc.toCreateCache[setName] + _, ok2 := dc.toUpdateCache[setName] + return ok1 || ok2 } -func (dc *memberAwareDirtyCache) isSetToDelete(setName string) bool { - _, ok := dc.toDeleteCache[setName] +func (dc *dirtyCache) isSetToDelete(setName string) bool { + _, ok := dc.toDestroyCache[setName] return ok } -func (dc *memberAwareDirtyCache) printAddOrUpdateCache() string { - return fmt.Sprintf("%+v", dc.toAddOrUpdateCache) +func (dc *dirtyCache) printAddOrUpdateCache() string { + return fmt.Sprintf("[to create: %+v], [to update: %+v]", dc.toCreateCache, dc.toUpdateCache) } -func (dc *memberAwareDirtyCache) printDeleteCache() string { - return fmt.Sprintf("%+v", dc.toDeleteCache) +func (dc *dirtyCache) printDeleteCache() string { + return fmt.Sprintf("%+v", dc.toDestroyCache) } -func (dc *memberAwareDirtyCache) getOriginalMembers(setName string) map[string]struct{} { - members, ok := dc.toAddOrUpdateCache[setName] - if !ok { - members, ok = dc.toDeleteCache[setName] - if !ok { - return nil - } +func (dc *dirtyCache) getMembersToAdd(setName string) map[string]struct{} { + fmt.Println("hey1") + diff, ok := dc.toCreateCache[setName] + if ok { + return diff.membersToAdd + } + fmt.Println("hey2") + diff, ok = dc.toUpdateCache[setName] + if ok { + return diff.membersToAdd + } + fmt.Println("hey3") + diff, ok = dc.toDestroyCache[setName] + if ok { + return diff.membersToAdd + } + return nil +} + +func (dc *dirtyCache) getMembersToDelete(setName string) map[string]struct{} { + diff, ok := dc.toCreateCache[setName] + if ok { + return diff.membersToDelete + } + diff, ok = dc.toUpdateCache[setName] + if ok { + return diff.membersToDelete + } + diff, ok = dc.toDestroyCache[setName] + if ok { + return diff.membersToDelete } - return members + return nil } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_test.go b/npm/pkg/dataplane/ipsets/dirtycache_test.go index f8773a56ba..aa7acabf19 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_test.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_test.go @@ -9,10 +9,15 @@ import ( // members are only important for Linux type dirtyCacheResults struct { - // map of prefixed name to members - toAddOrUpdate map[string][]string - // map of prefixed name to members - toDelete map[string][]string + // map of prefixed name to members to add/delete + toAddOrUpdate map[string]testDiff + // map of prefixed name to members to add/delete + toDestroy map[string]testDiff +} + +type testDiff struct { + toAdd []string + toDelete []string } const ( @@ -23,164 +28,173 @@ const ( func TestDirtyCacheReset(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) set2 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + set3 := NewIPSet(NewIPSetMetadata("set3", Namespace)) + set4 := NewIPSet(NewIPSetMetadata("set4", Namespace)) dc := newDirtyCache() - set1.IPPodKey[ip] = podKey dc.create(set1) - dc.delete(set2) + dc.addMember(set2, ip) + dc.deleteMember(set3, "4.4.4.4") + dc.destroy(set4) dc.reset() assertDirtyCache(t, dc, &dirtyCacheResults{}) } func TestDirtyCacheCreate(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - set2 := NewIPSet(NewIPSetMetadata("set2", Namespace)) - dc := newDirtyCache() - dc.create(set1) - // members are ignored on create - set2.IPPodKey[ip] = podKey - dc.create(set2) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ - set1.Name: {}, - set2.Name: {}, - }, - toDelete: nil, - }) -} - -func TestDirtyCacheCreateAfterDelete(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() - set1.IPPodKey[ip] = podKey - dc.delete(set1) - // original members shouldn't get updated dc.create(set1) assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ - set1.Name: {ip}, - }, - toDelete: nil, - }) -} - -func TestDirtyCacheUpdateNew(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - set1.IPPodKey[ip] = podKey - dc.update(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ - set1.Name: {ip}, - }, - toDelete: nil, - }) -} - -func TestDirtyCacheUpdateOld(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - dc.create(set1) - // original members shouldn't get updated - set1.IPPodKey[ip] = podKey - dc.update(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ - set1.Name: {}, - }, - toDelete: nil, - }) -} - -func TestDirtyCacheUpdateTwice(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - dc.update(set1) - // original members shouldn't get updated - set1.IPPodKey[ip] = podKey - dc.update(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ + toAddOrUpdate: map[string]testDiff{ set1.Name: {}, }, - toDelete: nil, + toDestroy: nil, }) } -func TestDirtyCacheUpdateAfterDelete(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - dc.delete(set1) - // original members shouldn't get updated - set1.IPPodKey[ip] = podKey - dc.update(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string][]string{ - set1.Name: {}, - }, - toDelete: nil, - }) -} - -func TestDirtyCacheDelete(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - set1.IPPodKey[ip] = podKey - dc.delete(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: nil, - toDelete: map[string][]string{ - set1.Name: {ip}, - }, - }) -} - -func TestDirtyCacheDeleteOld(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - set1.IPPodKey[ip] = podKey - dc.update(set1) - // original members shouldn't get updated - delete(set1.IPPodKey, ip) - dc.delete(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: nil, - toDelete: map[string][]string{ - set1.Name: {ip}, - }, - }) -} - -func TestDirtyCacheDeleteTwice(t *testing.T) { - set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) - dc := newDirtyCache() - set1.IPPodKey[ip] = podKey - dc.update(set1) - // original members shouldn't get updated - delete(set1.IPPodKey, ip) - dc.delete(set1) - assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: nil, - toDelete: map[string][]string{ - set1.Name: {ip}, - }, - }) -} - -func assertDirtyCache(t *testing.T, dc dirtyCacheMaintainer, expected *dirtyCacheResults) { +// func TestDirtyCacheCreateAfterDelete(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// set1.IPPodKey[ip] = podKey +// dc.destroy(set1) +// // original members shouldn't get updated +// dc.create(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: map[string]*memberDiff{ +// set1.Name: {ip}, +// }, +// toDestroy: nil, +// }) +// } + +// func TestDirtyCacheUpdateNew(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: map[string][]string{ +// set1.Name: {ip}, +// }, +// toDestroy: nil, +// }) +// } + +// func TestDirtyCacheUpdateOld(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// dc.create(set1) +// // original members shouldn't get updated +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: map[string][]string{ +// set1.Name: {}, +// }, +// toDestroy: nil, +// }) +// } + +// func TestDirtyCacheUpdateTwice(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// dc.update(set1) +// // original members shouldn't get updated +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: map[string][]string{ +// set1.Name: {}, +// }, +// toDestroy: nil, +// }) +// } + +// func TestDirtyCacheUpdateAfterDelete(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// dc.destroy(set1) +// // original members shouldn't get updated +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: map[string][]string{ +// set1.Name: {}, +// }, +// toDestroy: nil, +// }) +// } + +// func TestDirtyCacheDelete(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// set1.IPPodKey[ip] = podKey +// dc.destroy(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: nil, +// toDestroy: map[string][]string{ +// set1.Name: {ip}, +// }, +// }) +// } + +// func TestDirtyCacheDeleteOld(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// // original members shouldn't get updated +// delete(set1.IPPodKey, ip) +// dc.destroy(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: nil, +// toDestroy: map[string][]string{ +// set1.Name: {ip}, +// }, +// }) +// } + +// func TestDirtyCacheDeleteTwice(t *testing.T) { +// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) +// dc := newDirtyCache() +// set1.IPPodKey[ip] = podKey +// dc.update(set1) +// // original members shouldn't get updated +// delete(set1.IPPodKey, ip) +// dc.destroy(set1) +// assertDirtyCache(t, dc, &dirtyCacheResults{ +// toAddOrUpdate: nil, +// toDestroy: map[string][]string{ +// set1.Name: {ip}, +// }, +// }) +// } + +func assertDirtyCache(t *testing.T, dc dirtyCacheInterface, expected *dirtyCacheResults) { require.Equal(t, len(expected.toAddOrUpdate), dc.numSetsToAddOrUpdate(), "unexpected number of sets to add or update") - require.Equal(t, len(expected.toDelete), dc.numSetsToDelete(), "unexpected number of sets to delete") - for setName, members := range expected.toAddOrUpdate { + require.Equal(t, len(expected.toDestroy), dc.numSetsToDelete(), "unexpected number of sets to delete") + for setName, diff := range expected.toAddOrUpdate { require.True(t, dc.isSetToAddOrUpdate(setName), "set %s should be added/updated", setName) require.False(t, dc.isSetToDelete(setName), "set %s should not be deleted", setName) - if !util.IsWindowsDP() { - require.Equal(t, stringSliceToSet(members), dc.getOriginalMembers(setName), "unexpected original members for set %s", setName) - } + assertDiff(t, dc, setName, diff) } - for setName, members := range expected.toDelete { + for setName, diff := range expected.toDestroy { require.True(t, dc.isSetToDelete(setName), "set %s should be deleted", setName) require.False(t, dc.isSetToAddOrUpdate(setName), "set %s should not be added/updated", setName) - if !util.IsWindowsDP() { - require.Equal(t, stringSliceToSet(members), dc.getOriginalMembers(setName), "unexpected original members for set %s", setName) + assertDiff(t, dc, setName, diff) + } +} + +func assertDiff(t *testing.T, dc dirtyCacheInterface, setName string, diff testDiff) { + if !util.IsWindowsDP() { + if len(diff.toAdd) == 0 { + require.Equal(t, 0, len(dc.getMembersToAdd(setName)), "expected 0 members to add") + } else { + require.Equal(t, stringSliceToSet(diff.toAdd), dc.getMembersToAdd(setName), "unexpected members to add for set %s", setName) + } + if len(diff.toDelete) == 0 { + require.Equal(t, 0, len(dc.getMembersToDelete(setName)), "expected 0 members to delete") + } else { + require.Equal(t, stringSliceToSet(diff.toDelete), dc.getMembersToDelete(setName), "unexpected members to delete for set %s", setName) } } } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 0cf32b7adc..d9d0226618 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -2,36 +2,38 @@ package ipsets import "fmt" -type nameOnlyDirtyCache struct { +type dirtyCache struct { toAddOrUpdateCache map[string]struct{} toDeleteCache map[string]struct{} } func newDirtyCache() dirtyCacheMaintainer { - return &nameOnlyDirtyCache{ - toAddOrUpdateCache: make(map[string]struct{}), - toDeleteCache: make(map[string]struct{}), - } + dc := &dirtyCache{} + dc.reset() + return dc } -func (dc *nameOnlyDirtyCache) reset() { +func (dc *dirtyCache) reset() { dc.toAddOrUpdateCache = make(map[string]struct{}) dc.toDeleteCache = make(map[string]struct{}) } -func (dc *nameOnlyDirtyCache) create(newSet *IPSet) { - putIntoAndRemoveFromOther(newSet, dc.toAddOrUpdateCache, dc.toDeleteCache) +func (dc *dirtyCache) create(set *IPSet) { + putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) +} +func (dc *dirtyCache) addMember(set *IPSet, member string) { + putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) } -func (dc *nameOnlyDirtyCache) update(originalSet *IPSet) { - putIntoAndRemoveFromOther(originalSet, dc.toAddOrUpdateCache, dc.toDeleteCache) +func (dc *dirtyCache) deleteMember(set *IPSet, member string) { + putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) } -func (dc *nameOnlyDirtyCache) delete(originalSet *IPSet) { - putIntoAndRemoveFromOther(originalSet, dc.toDeleteCache, dc.toAddOrUpdateCache) +func (dc *dirtyCache) destroy(set *IPSet) { + putInCacheAndRemoveFromOther(set, dc.toDeleteCache, dc.toAddOrUpdateCache) } -func putIntoAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struct{}) { +func putInCacheAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struct{}) { if _, ok := intoCache[set.Name]; ok { return } @@ -39,7 +41,7 @@ func putIntoAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struc delete(fromCache, set.Name) } -func (dc *nameOnlyDirtyCache) getSetsToAddOrUpdate() []string { +func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { result := make([]string, 0, len(dc.toAddOrUpdateCache)) for setName := range dc.toAddOrUpdateCache { result = append(result, setName) @@ -47,7 +49,7 @@ func (dc *nameOnlyDirtyCache) getSetsToAddOrUpdate() []string { return result } -func (dc *nameOnlyDirtyCache) getSetsToDelete() []string { +func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { result := make([]string, 0, len(dc.toDeleteCache)) for setName := range dc.toDeleteCache { result = append(result, setName) @@ -55,32 +57,36 @@ func (dc *nameOnlyDirtyCache) getSetsToDelete() []string { return result } -func (dc *nameOnlyDirtyCache) numSetsToAddOrUpdate() int { +func (dc *dirtyCache) numSetsToAddOrUpdate() int { return len(dc.toAddOrUpdateCache) } -func (dc *nameOnlyDirtyCache) numSetsToDelete() int { +func (dc *dirtyCache) numSetsToDelete() int { return len(dc.toDeleteCache) } -func (dc *nameOnlyDirtyCache) isSetToAddOrUpdate(setName string) bool { +func (dc *dirtyCache) isSetToAddOrUpdate(setName string) bool { _, ok := dc.toAddOrUpdateCache[setName] return ok } -func (dc *nameOnlyDirtyCache) isSetToDelete(setName string) bool { +func (dc *dirtyCache) isSetToDelete(setName string) bool { _, ok := dc.toDeleteCache[setName] return ok } -func (dc *nameOnlyDirtyCache) printAddOrUpdateCache() string { +func (dc *dirtyCache) printAddOrUpdateCache() string { return fmt.Sprintf("%+v", dc.toAddOrUpdateCache) } -func (dc *nameOnlyDirtyCache) printDeleteCache() string { +func (dc *dirtyCache) printDeleteCache() string { return fmt.Sprintf("%+v", dc.toDeleteCache) } -func (dc *nameOnlyDirtyCache) getOriginalMembers(_ string) map[string]struct{} { +func (dc *dirtyCache) getMembersToAdd(setName string) map[string]struct{} { + return nil +} + +func (dc *dirtyCache) getMembersToDelete(setName string) map[string]struct{} { return nil } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index abc943aab1..a44574f14e 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -36,7 +36,7 @@ const ( type IPSetManager struct { iMgrCfg *IPSetManagerCfg setMap map[string]*IPSet - dirtyCache dirtyCacheMaintainer + dirtyCache dirtyCacheInterface ioShim *common.IOShim sync.Mutex } @@ -231,8 +231,7 @@ func (iMgr *IPSetManager) AddToSets(addToSets []*IPSetMetadata, ip, podKey strin // 2. add ip to the set, and update the pod key _, ok := set.IPPodKey[ip] if !ok { - // needs to be called before updating the cache - iMgr.modifyCacheForKernelMemberUpdate(set) + iMgr.modifyCacheForKernelMemberAdd(set, ip) metrics.AddEntryToIPSet(prefixedName) } set.IPPodKey[ip] = podKey @@ -281,10 +280,8 @@ func (iMgr *IPSetManager) RemoveFromSets(removeFromSets []*IPSetMetadata, ip, po continue } - // needs to be called before updating the cache - iMgr.modifyCacheForKernelMemberUpdate(set) - // update the IP ownership with podkey + iMgr.modifyCacheForKernelMemberDelete(set, ip) delete(set.IPPodKey, ip) metrics.RemoveEntryFromIPSet(prefixedName) } @@ -336,10 +333,7 @@ func (iMgr *IPSetManager) AddToLists(listMetadatas, setMetadatas []*IPSetMetadat } member := iMgr.setMap[memberName] - // needs to be called before updating the cache - // second call, third call, etc. for a given list is idempotent - iMgr.modifyCacheForKernelMemberUpdate(list) - + iMgr.modifyCacheForKernelMemberAdd(list, member.HashedName) list.MemberIPSets[memberName] = member member.incIPSetReferCount() metrics.AddEntryToIPSet(list.Name) @@ -396,10 +390,7 @@ func (iMgr *IPSetManager) RemoveFromList(listMetadata *IPSetMetadata, setMetadat continue } - // needs to be called before updating the cache - // second call, third call, etc. for a given list is idempotent - iMgr.modifyCacheForKernelMemberUpdate(list) - + iMgr.modifyCacheForKernelMemberDelete(list, member.HashedName) delete(list.MemberIPSets, memberName) member.decIPSetReferCount() metrics.RemoveEntryFromIPSet(list.Name) @@ -503,7 +494,7 @@ func (iMgr *IPSetManager) shouldBeInKernel(set *IPSet) bool { } func (iMgr *IPSetManager) modifyCacheForKernelRemoval(set *IPSet) { - iMgr.dirtyCache.delete(set) + iMgr.dirtyCache.destroy(set) /* TODO kernel-based prometheus metrics @@ -520,11 +511,15 @@ func (iMgr *IPSetManager) decKernelReferCountAndModifyCache(member *IPSet) { } } -// need to call this before updating the set -// can be called multiple times -func (iMgr *IPSetManager) modifyCacheForKernelMemberUpdate(set *IPSet) { +func (iMgr *IPSetManager) modifyCacheForKernelMemberAdd(set *IPSet, member string) { + if iMgr.shouldBeInKernel(set) { + iMgr.dirtyCache.addMember(set, member) + } +} + +func (iMgr *IPSetManager) modifyCacheForKernelMemberDelete(set *IPSet, member string) { if iMgr.shouldBeInKernel(set) { - iMgr.dirtyCache.update(set) + iMgr.dirtyCache.deleteMember(set, member) } } @@ -532,7 +527,7 @@ func (iMgr *IPSetManager) modifyCacheForKernelMemberUpdate(set *IPSet) { // if so will not delete it func (iMgr *IPSetManager) sanitizeDirtyCache() { anyProblems := false - for _, setName := range iMgr.dirtyCache.getSetsToDelete() { + for setName := range iMgr.dirtyCache.getSetsToDelete() { if iMgr.dirtyCache.isSetToAddOrUpdate(setName) { klog.Errorf("[IPSetManager] Unexpected state in dirty cache %s set is part of both update and delete caches", setName) anyProblems = true diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 662fde2c3c..8f60a874bc 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -2,6 +2,7 @@ package ipsets import ( "fmt" + "strings" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/parse" "github.com/Azure/azure-container-networking/npm/util" @@ -179,8 +180,6 @@ func (iMgr *IPSetManager) fileCreatorForReset(ipsetListOutput []byte) (*ioutil.F } /* -FIXME update documentation - overall error handling for ipset restore file. ipset restore will apply all lines to the kernel before a failure, so when recovering from a line failure, we must skip the lines that were already applied. below, "set" refers to either hashset or list, except in the sections for adding to (hash)set and adding to list @@ -245,19 +244,58 @@ example where every set in add/update cache should have ip 1.2.3.4 and 2.3.4.5: -X set-to-delete2 -X set-to-delete3 -X set-to-delete1 +*/ +func (iMgr *IPSetManager) applyIPSetsWithSaveFile() error { + var saveFile []byte + var saveError error + if iMgr.dirtyCache.numSetsToAddOrUpdate() > 0 { + saveFile, saveError = iMgr.ipsetSave() + if saveError != nil { + return npmerrors.SimpleErrorWrapper("ipset save failed when applying ipsets", saveError) + } + } + creator := iMgr.fileCreatorForApplyWithSaveFile(maxTryCount, saveFile) + restoreError := creator.RunCommandWithFile(ipsetCommand, ipsetRestoreFlag) + if restoreError != nil { + return npmerrors.SimpleErrorWrapper("ipset restore failed when applying ipsets with save file", restoreError) + } + return nil +} + +/* +See error handling in applyIPSetsWithSaveFile(). + +overall format for ipset restore file: + [creates] (random order) + [deletes and adds] (sets in random order, where each set has deletes first (random order), then adds (random order)) + [flushes] (random order) + [destroys] (random order) + +example where: +- set1 and set2 will delete 1.2.3.4 and 2.3.4.5 and add 7.7.7.7 and 8.8.8.8 +- set3 will be created with 1.0.0.1 +- set4 and set5 will be destroyed + restore file: [flag meanings: -F (flush), -X (destroy), -N (create), -D (delete), -A (add)] + -N set2 + -N set3 + -N set1 + -D set2 2.3.4.5 + -D set2 1.2.3.4 + -A set2 8.8.8.8 + -A set2 7.7.7.7 + -A set3 1.0.0.1 + -D set1 1.2.3.4 + -D set1 2.3.4.5 + -A set1 7.7.7.7 + -A set1 8.8.8.8 + -F set5 + -F set4 + -X set5 + -X set4 */ func (iMgr *IPSetManager) applyIPSets() error { - var saveFile []byte - // FIXME delete - // var saveError error - // if iMgr.dirtyCache.numSetsToAddOrUpdate() > 0 { - // saveFile, saveError = iMgr.ipsetSave() - // if saveError != nil { - // return npmerrors.SimpleErrorWrapper("ipset save failed when applying ipsets", saveError) - // } - // } - creator := iMgr.fileCreatorForApply(maxTryCount, saveFile) + creator := iMgr.fileCreatorForApply(maxTryCount) restoreError := creator.RunCommandWithFile(ipsetCommand, ipsetRestoreFlag) if restoreError != nil { return npmerrors.SimpleErrorWrapper("ipset restore failed when applying ipsets", restoreError) @@ -278,53 +316,79 @@ func (iMgr *IPSetManager) ipsetSave() ([]byte, error) { return saveFile, nil } -// FIXME remove saveFile param -func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) *ioutil.FileCreator { +// NOTE: duplicate code in the first step of this function and fileCreatorForApply +func (iMgr *IPSetManager) fileCreatorForApplyWithSaveFile(maxTryCount int, saveFile []byte) *ioutil.FileCreator { creator := ioutil.NewFileCreator(iMgr.ioShim, maxTryCount, ipsetRestoreLineFailurePattern) // TODO make the line failure pattern into a definition constant eventually // 1. create all sets first so we don't try to add a member set to a list if it hasn't been created yet setsToAddOrUpdate := iMgr.dirtyCache.getSetsToAddOrUpdate() - for _, prefixedName := range setsToAddOrUpdate { + for prefixedName := range setsToAddOrUpdate { set := iMgr.setMap[prefixedName] iMgr.createSetForApply(creator, set) // NOTE: currently no logic to handle this scenario: // if a set in the toAddOrUpdateCache is in the kernel with the wrong type, then we'll try to create it, which will fail in the first restore call, but then be skipped in a retry } - // 2. add/delete members from dirty sets to add or update - for _, prefixedName := range setsToAddOrUpdate { - sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) - originalMembers := iMgr.dirtyCache.getOriginalMembers(prefixedName) + // 2. for dirty sets already in the kernel, update members (add members not in the kernel, and delete undesired members in the kernel) + iMgr.updateDirtyKernelSets(setsToAddOrUpdate, saveFile, creator) + + // 3. for the remaining dirty sets, add their members to the kernel + for prefixedName := range setsToAddOrUpdate { set := iMgr.setMap[prefixedName] + sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) if set.Kind == HashSet { for ip := range set.IPPodKey { - if _, ok := originalMembers[ip]; ok { - // remove from members so we don't try to delete it later - delete(originalMembers, ip) - } else { - // add the member since it didn't exist before - iMgr.addMemberForApply(creator, set, sectionID, ip) - } + iMgr.addMemberForApply(creator, set, sectionID, ip) } } else { - for memberName, memberSet := range set.MemberIPSets { - if _, ok := originalMembers[memberName]; ok { - // remove from members so we don't try to delete it later - delete(originalMembers, memberName) - } else { - // add the member since it didn't exist before - iMgr.addMemberForApply(creator, set, sectionID, memberSet.HashedName) - } + for _, member := range set.MemberIPSets { + iMgr.addMemberForApply(creator, set, sectionID, member.HashedName) } } - // originalMembers now contains only the members that need to be deleted - for member := range originalMembers { - if set.Kind == HashSet { - iMgr.deleteMemberForApply(creator, set, sectionID, member) - } else { - memberSet := iMgr.setMap[member] - iMgr.deleteMemberForApply(creator, set, sectionID, memberSet.HashedName) - } + } + + /* + 4. flush and destroy sets in the original delete cache + We must perform this step after member deletions because of the following scenario: + Suppose we want to destroy set A, which is referenced by list L. For set A to be in the toDeleteCache, + we must have deleted the reference in list L, so list L is in the toAddOrUpdateCache. In step 2, we will delete this reference, + but until then, set A is in use by a kernel component and can't be destroyed. + */ + // flush all sets first in case a set we're destroying is referenced by a list we're destroying + setsToDelete := iMgr.dirtyCache.getSetsToDelete() + for prefixedName := range setsToDelete { + iMgr.flushSetForApply(creator, prefixedName) + } + for prefixedName := range setsToDelete { + iMgr.destroySetForApply(creator, prefixedName) + } + return creator +} + +// NOTE: duplicate code in the first step in this function and fileCreatorForApplyWithSaveFile +func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int) *ioutil.FileCreator { + creator := ioutil.NewFileCreator(iMgr.ioShim, maxTryCount, ipsetRestoreLineFailurePattern) // TODO make the line failure pattern into a definition constant eventually + + // 1. create all sets first so we don't try to add a member set to a list if it hasn't been created yet + setsToAddOrUpdate := iMgr.dirtyCache.getSetsToAddOrUpdate() + for prefixedName := range setsToAddOrUpdate { + set := iMgr.setMap[prefixedName] + iMgr.createSetForApply(creator, set) + // NOTE: currently no logic to handle this scenario: + // if a set in the toAddOrUpdateCache is in the kernel with the wrong type, then we'll try to create it, which will fail in the first restore call, but then be skipped in a retry + } + + // 2. delete/add members from dirty sets to add or update + for prefixedName := range setsToAddOrUpdate { + sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) + set := iMgr.setMap[prefixedName] + membersToDelete := iMgr.dirtyCache.getMembersToDelete(prefixedName) + for member := range membersToDelete { + iMgr.deleteMemberForApply(creator, set, sectionID, member) + } + membersToAdd := iMgr.dirtyCache.getMembersToAdd(prefixedName) + for member := range membersToAdd { + iMgr.addMemberForApply(creator, set, sectionID, member) } } @@ -338,15 +402,176 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int, saveFile []byte) */ // flush all sets first in case a set we're destroying is referenced by a list we're destroying setsToDelete := iMgr.dirtyCache.getSetsToDelete() - for _, prefixedName := range setsToDelete { + for prefixedName := range setsToDelete { iMgr.flushSetForApply(creator, prefixedName) } - for _, prefixedName := range setsToDelete { + for prefixedName := range setsToDelete { iMgr.destroySetForApply(creator, prefixedName) } return creator } +// updates the creator (adds/deletes members) for dirty sets already in the kernel +// updates the setsToAddOrUpdate: after calling this function, the map will only consist of sets to create +// error handling principal: +// - if contract with ipset save (or grep) is breaking, salvage what we can, take a snapshot (TODO), and log the failure +// - have a background process for sending/removing snapshots intermittently +func (iMgr *IPSetManager) updateDirtyKernelSets(setsToAddOrUpdate map[string]struct{}, saveFile []byte, creator *ioutil.FileCreator) { + // map hashed names to prefixed names + toAddOrUpdateHashedNames := make(map[string]string) + for prefixedName := range setsToAddOrUpdate { + hashedName := iMgr.setMap[prefixedName].HashedName + toAddOrUpdateHashedNames[hashedName] = prefixedName + } + + // in each iteration, read a create line and any ensuing add lines + readIndex := 0 + var line []byte + if readIndex < len(saveFile) { + line, readIndex = parse.Line(readIndex, saveFile) + if !hasPrefix(line, createStringWithSpace) { + klog.Errorf("expected a create line in ipset save file, but got the following line: %s", string(line)) + // TODO send error snapshot + line, readIndex = nextCreateLine(readIndex, saveFile) + } + } + for readIndex < len(saveFile) { + // 1. get the hashed name + lineAfterCreate := string(line[len(createStringWithSpace):]) + spaceSplitLineAfterCreate := strings.Split(lineAfterCreate, space) + hashedName := spaceSplitLineAfterCreate[0] + + // 2. continue to the next create line if the set isn't in the toAddOrUpdateCache + prefixedName, shouldModify := toAddOrUpdateHashedNames[hashedName] + if !shouldModify { + line, readIndex = nextCreateLine(readIndex, saveFile) + continue + } + + // 3. update the set from the kernel + set := iMgr.setMap[prefixedName] + // remove from the dirty cache so we don't add it later + delete(setsToAddOrUpdate, prefixedName) + // mark the set as in the kernel + delete(toAddOrUpdateHashedNames, hashedName) + + // 3.1 check for consistent type + restOfLine := spaceSplitLineAfterCreate[1:] + if haveTypeProblem(set, restOfLine) { + // error logging happens in the helper function + // TODO send error snapshot + line, readIndex = nextCreateLine(readIndex, saveFile) + continue + } + + // 3.2 get desired members from cache + var membersToAdd map[string]struct{} + if set.Kind == HashSet { + membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) + for ip := range set.IPPodKey { + membersToAdd[ip] = struct{}{} + } + } else { + membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) + for _, member := range set.MemberIPSets { + membersToAdd[member.HashedName] = struct{}{} + } + } + + // 3.4 determine which members to add/delete + membersToDelete := make(map[string]struct{}) + for readIndex < len(saveFile) { + line, readIndex = parse.Line(readIndex, saveFile) + if hasPrefix(line, createStringWithSpace) { + break + } + if !hasPrefix(line, addStringWithSpace) { + klog.Errorf("expected an add line, but got the following line: %s", string(line)) + // TODO send error snapshot + line, readIndex = nextCreateLine(readIndex, saveFile) + break + } + lineAfterAdd := string(line[len(addStringWithSpace):]) + spaceSplitLineAfterAdd := strings.Split(lineAfterAdd, space) + parent := spaceSplitLineAfterAdd[0] + if len(spaceSplitLineAfterAdd) != 2 || parent != hashedName { + klog.Errorf("expected an add line for set %s in ipset save file, but got the following line: %s", hashedName, string(line)) + // TODO send error snapshot + line, readIndex = nextCreateLine(readIndex, saveFile) + break + } + member := spaceSplitLineAfterAdd[1] + + _, shouldKeep := membersToAdd[member] + if shouldKeep { + // member already in the kernel, so don't add it later + delete(membersToAdd, member) + } else { + // member should be deleted from the kernel + membersToDelete[member] = struct{}{} + } + } + + // 3.5 delete undesired members from restore file + sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) + for member := range membersToDelete { + iMgr.deleteMemberForApply(creator, set, sectionID, member) + } + // 3.5 add new members to restore file + for member := range membersToAdd { + iMgr.addMemberForApply(creator, set, sectionID, member) + } + } +} + +func nextCreateLine(originalReadIndex int, saveFile []byte) (createLine []byte, nextReadIndex int) { + nextReadIndex = originalReadIndex + for nextReadIndex < len(saveFile) { + createLine, nextReadIndex = parse.Line(nextReadIndex, saveFile) + if hasPrefix(createLine, createStringWithSpace) { + return + } + } + return +} + +func haveTypeProblem(set *IPSet, restOfSpaceSplitCreateLine []string) bool { + // TODO check type based on maxelem for hash sets? CIDR blocks have a different maxelem + if len(restOfSpaceSplitCreateLine) == 0 { + klog.Error("expected a type specification for the create line but received nothing after the set name") + return true + } + typeString := restOfSpaceSplitCreateLine[0] + switch typeString { + case ipsetSetListString: + if set.Kind != ListSet { + lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) + klog.Errorf("expected to find a ListSet but have the line: %s", lineString) + return true + } + case ipsetNetHashString: + if set.Kind != HashSet || set.Type == NamedPorts { + lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) + klog.Errorf("expected to find a non-NamedPorts HashSet but have the following line: %s", lineString) + return true + } + case ipsetIPPortHashString: + if set.Type != NamedPorts { + lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) + klog.Errorf("expected to find a NamedPorts set but have the following line: %s", lineString) + return true + } + default: + klog.Errorf("unknown type string [%s] in line: %s", typeString, strings.Join(restOfSpaceSplitCreateLine, " ")) + return true + } + return false +} + +func hasPrefix(line []byte, prefix string) bool { + return len(line) >= len(prefix) && string(line[:len(prefix)]) == prefix +} + func (iMgr *IPSetManager) flushSetForApply(creator *ioutil.FileCreator, prefixedName string) { errorHandlers := []*ioutil.LineErrorHandler{ { @@ -480,164 +705,3 @@ func (iMgr *IPSetManager) addMemberForApply(creator *ioutil.FileCreator, set *IP func sectionID(prefix, prefixedName string) string { return fmt.Sprintf("%s-%s", prefix, prefixedName) } - -// updates the creator (adds/deletes members) for dirty sets already in the kernel -// updates the toAddOrUpdateCache: after calling this function, the cache will only consist of sets to create -// error handling principal: -// - if contract with ipset save (or grep) is breaking, salvage what we can, take a snapshot (TODO), and log the failure -// - have a background process for sending/removing snapshots intermittently -// func (iMgr *IPSetManager) updateDirtyKernelSets(saveFile []byte, creator *ioutil.FileCreator) { -// // map hashed names to prefixed names -// toAddOrUpdateHashedNames := make(map[string]string) -// for prefixedName := range iMgr.toAddOrUpdateCache { -// hashedName := iMgr.setMap[prefixedName].HashedName -// toAddOrUpdateHashedNames[hashedName] = prefixedName -// } - -// // in each iteration, read a create line and any ensuing add lines -// readIndex := 0 -// var line []byte -// if readIndex < len(saveFile) { -// line, readIndex = parse.Line(readIndex, saveFile) -// if !hasPrefix(line, createStringWithSpace) { -// klog.Errorf("expected a create line in ipset save file, but got the following line: %s", string(line)) -// // TODO send error snapshot -// line, readIndex = nextCreateLine(readIndex, saveFile) -// } -// } -// for readIndex < len(saveFile) { -// // 1. get the hashed name -// lineAfterCreate := string(line[len(createStringWithSpace):]) -// spaceSplitLineAfterCreate := strings.Split(lineAfterCreate, space) -// hashedName := spaceSplitLineAfterCreate[0] - -// // 2. continue to the next create line if the set isn't in the toAddOrUpdateCache -// prefixedName, shouldModify := toAddOrUpdateHashedNames[hashedName] -// if !shouldModify { -// line, readIndex = nextCreateLine(readIndex, saveFile) -// continue -// } - -// // 3. update the set from the kernel -// set := iMgr.setMap[prefixedName] -// // remove from the dirty cache so we don't add it later -// delete(iMgr.toAddOrUpdateCache, prefixedName) -// // mark the set as in the kernel -// delete(toAddOrUpdateHashedNames, hashedName) - -// // 3.1 check for consistent type -// restOfLine := spaceSplitLineAfterCreate[1:] -// if haveTypeProblem(set, restOfLine) { -// // error logging happens in the helper function -// // TODO send error snapshot -// line, readIndex = nextCreateLine(readIndex, saveFile) -// continue -// } - -// // 3.2 get desired members from cache -// var membersToAdd map[string]struct{} -// if set.Kind == HashSet { -// membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) -// for ip := range set.IPPodKey { -// membersToAdd[ip] = struct{}{} -// } -// } else { -// membersToAdd = make(map[string]struct{}, len(set.IPPodKey)) -// for _, member := range set.MemberIPSets { -// membersToAdd[member.HashedName] = struct{}{} -// } -// } - -// // 3.4 determine which members to add/delete -// membersToDelete := make(map[string]struct{}) -// for readIndex < len(saveFile) { -// line, readIndex = parse.Line(readIndex, saveFile) -// if hasPrefix(line, createStringWithSpace) { -// break -// } -// if !hasPrefix(line, addStringWithSpace) { -// klog.Errorf("expected an add line, but got the following line: %s", string(line)) -// // TODO send error snapshot -// line, readIndex = nextCreateLine(readIndex, saveFile) -// break -// } -// lineAfterAdd := string(line[len(addStringWithSpace):]) -// spaceSplitLineAfterAdd := strings.Split(lineAfterAdd, space) -// parent := spaceSplitLineAfterAdd[0] -// if len(spaceSplitLineAfterAdd) != 2 || parent != hashedName { -// klog.Errorf("expected an add line for set %s in ipset save file, but got the following line: %s", hashedName, string(line)) -// // TODO send error snapshot -// line, readIndex = nextCreateLine(readIndex, saveFile) -// break -// } -// member := spaceSplitLineAfterAdd[1] - -// _, shouldKeep := membersToAdd[member] -// if shouldKeep { -// // member already in the kernel, so don't add it later -// delete(membersToAdd, member) -// } else { -// // member should be deleted from the kernel -// membersToDelete[member] = struct{}{} -// } -// } - -// // 3.5 delete undesired members from restore file -// sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) -// for member := range membersToDelete { -// iMgr.deleteMemberForApply(creator, set, sectionID, member) -// } -// // 3.5 add new members to restore file -// for member := range membersToAdd { -// iMgr.addMemberForApply(creator, set, sectionID, member) -// } -// } -// } - -// func nextCreateLine(originalReadIndex int, saveFile []byte) (createLine []byte, nextReadIndex int) { -// nextReadIndex = originalReadIndex -// for nextReadIndex < len(saveFile) { -// createLine, nextReadIndex = parse.Line(nextReadIndex, saveFile) -// if hasPrefix(createLine, createStringWithSpace) { -// return -// } -// } -// return -// } - -// func haveTypeProblem(set *IPSet, restOfSpaceSplitCreateLine []string) bool { -// // TODO check type based on maxelem for hash sets? CIDR blocks have a different maxelem -// if len(restOfSpaceSplitCreateLine) == 0 { -// klog.Error("expected a type specification for the create line but received nothing after the set name") -// return true -// } -// typeString := restOfSpaceSplitCreateLine[0] -// switch typeString { -// case ipsetSetListString: -// if set.Kind != ListSet { -// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) -// klog.Errorf("expected to find a ListSet but have the line: %s", lineString) -// return true -// } -// case ipsetNetHashString: -// if set.Kind != HashSet || set.Type == NamedPorts { -// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) -// klog.Errorf("expected to find a non-NamedPorts HashSet but have the following line: %s", lineString) -// return true -// } -// case ipsetIPPortHashString: -// if set.Type != NamedPorts { -// lineString := fmt.Sprintf("create %s %s", set.HashedName, strings.Join(restOfSpaceSplitCreateLine, " ")) -// klog.Errorf("expected to find a NamedPorts set but have the following line: %s", lineString) -// return true -// } -// default: -// klog.Errorf("unknown type string [%s] in line: %s", typeString, strings.Join(restOfSpaceSplitCreateLine, " ")) -// return true -// } -// return false -// } - -// func hasPrefix(line []byte, prefix string) bool { -// return len(line) >= len(prefix) && string(line[:len(prefix)]) == prefix -// } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go index 2100121e98..a8cff6fe97 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/npm/metrics/promutil" dptestutils "github.com/Azure/azure-container-networking/npm/pkg/dataplane/testutils" "github.com/Azure/azure-container-networking/npm/util" + "github.com/Azure/azure-container-networking/npm/util/ioutil" testutils "github.com/Azure/azure-container-networking/test/utils" "github.com/stretchr/testify/require" ) @@ -31,74 +32,9 @@ azure-npm-777777` var resetIPSetsListOutput = []byte(resetIPSetsListOutputString) -func TestIPSetSave(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, - {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - output, err := iMgr.ipsetSave() - require.NoError(t, err) - require.Equal(t, saveResult, string(output)) -} - -func TestIPSetSaveNoMatch(t *testing.T) { - calls := []testutils.TestCmd{ - {Cmd: ipsetSaveStringSlice, ExitCode: 1}, - {Cmd: []string{"grep", "azure-npm-"}}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - output, err := iMgr.ipsetSave() - require.NoError(t, err) - require.Nil(t, output) -} - -// FIXME delete -func TestFileCreator(t *testing.T) { - ioshim := common.NewMockIOShim(nil) - iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - - iMgr.CreateIPSets([]*IPSetMetadata{namespaceSet}) - creator := iMgr.fileCreatorForApply(0, nil) - fmt.Println(creator.ToString()) - - // actualLines := testAndSortRestoreFileString(t, creator.ToString()) - - // expectedLines := []string{ - // fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), - // fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), - // fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), - // fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), - // fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), - // fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - // fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), - // fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), - // fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), - // fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), - // fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), - // fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - // fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), - // fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), - // "", - // } - // sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - - // dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) - // wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") - // require.NoError(t, err, "ipset restore should be successful") - // require.False(t, wasFileAltered, "file should not be altered") -} - // TODO test that a reconcile list is updated for all the TestFailure UTs // TODO same exact TestFailure UTs for unknown errors -// FIXME uncomment func TestApplyIPSets(t *testing.T) { type args struct { toAddUpdateSets []*IPSetMetadata @@ -192,7 +128,7 @@ func TestApplyIPSets(t *testing.T) { iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) iMgr.CreateIPSets(tt.args.toAddUpdateSets) for _, set := range tt.args.toDeleteSets { - iMgr.dirtyCache.delete(NewIPSet(set)) + iMgr.dirtyCache.destroy(NewIPSet(set)) } err := iMgr.ApplyIPSets() @@ -220,6 +156,55 @@ func TestApplyIPSets(t *testing.T) { } } +func TestNextCreateLine(t *testing.T) { + createLine := "create test-list1 list:set size 8" + addLine := "add test-set1 1.2.3.4" + createLineWithNewline := createLine + "\n" + addLineWithNewline := addLine + "\n" + tests := []struct { + name string + lines []string + expectedReadIndex int + expectedLine []byte + }{ + // parse.Line will omit the newline at the end of the line unless it's the last line + { + name: "empty save file", + lines: []string{}, + expectedReadIndex: 0, + expectedLine: nil, + }, + { + name: "no creates", + lines: []string{addLineWithNewline}, + expectedReadIndex: len(addLineWithNewline), + expectedLine: []byte(addLineWithNewline), + }, + { + name: "start with create", + lines: []string{createLine, addLineWithNewline}, + expectedReadIndex: len(createLineWithNewline), + expectedLine: []byte(createLine), + }, + { + name: "create after adds", + lines: []string{addLine, addLine, createLineWithNewline}, + expectedReadIndex: 2*len(addLine+"\n") + len(createLine+"\n"), + expectedLine: []byte(createLineWithNewline), + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + saveFile := []byte(strings.Join(tt.lines, "\n")) + line, readIndex := nextCreateLine(0, saveFile) + require.Equal(t, tt.expectedReadIndex, readIndex) + require.Equal(t, tt.expectedLine, line) + }) + } + // fmt.Println(string([]byte(addLine + addLine, createLineWithNewline})[:78])) +} + func TestDestroyNPMIPSetsCreatorSuccess(t *testing.T) { calls := []testutils.TestCmd{fakeRestoreSuccessCommand} ioshim := common.NewMockIOShim(calls) @@ -473,7 +458,7 @@ func TestResetIPSetsOnFailure(t *testing.T) { }) } -func TestApplyIPSetsSuccess(t *testing.T) { +func TestApplyIPSetsSuccessWithoutSave(t *testing.T) { // no sets to add/update, so don't call ipset save calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} ioshim := common.NewMockIOShim(calls) @@ -487,6 +472,37 @@ func TestApplyIPSetsSuccess(t *testing.T) { require.NoError(t, err) } +func TestApplyIPSetsSuccessWithSave(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}}, + {Cmd: ipsetRestoreStringSlice}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err := iMgr.applyIPSetsWithSaveFile() + require.NoError(t, err) +} + +func TestApplyIPSetsFailureOnSave(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, HasStartError: true, PipedToCommand: true, ExitCode: 1}, + {Cmd: []string{"grep", "azure-npm-"}}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err := iMgr.applyIPSetsWithSaveFile() + require.Error(t, err) +} + func TestApplyIPSetsFailureOnRestore(t *testing.T) { calls := []testutils.TestCmd{ // fail 3 times because this is our max try count @@ -495,13 +511,29 @@ func TestApplyIPSetsFailureOnRestore(t *testing.T) { {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, } ioshim := common.NewMockIOShim(calls) - // defer ioshim.VerifyCalls(t, calls) + defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - // create a set so we run ipset save iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) err := iMgr.applyIPSets() require.Error(t, err) + + // same test with save file + calls = []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}}, + // fail 3 times because this is our max try count + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + } + ioshim = common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr = NewIPSetManager(applyAlwaysCfg, ioshim) + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err = iMgr.applyIPSetsWithSaveFile() + require.Error(t, err) } func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { @@ -512,30 +544,207 @@ func TestApplyIPSetsRecoveryForFailureOnRestore(t *testing.T) { ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - // create a set so we run ipset save iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) err := iMgr.applyIPSets() require.NoError(t, err) + + // same test with save file + calls = []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}}, + {Cmd: ipsetRestoreStringSlice, ExitCode: 1}, + {Cmd: ipsetRestoreStringSlice}, + } + ioshim = common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr = NewIPSetManager(applyAlwaysCfg, ioshim) + // create a set so we run ipset save + iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) + err = iMgr.applyIPSetsWithSaveFile() + require.NoError(t, err) +} + +func TestIPSetSave(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}, Stdout: saveResult}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + output, err := iMgr.ipsetSave() + require.NoError(t, err) + require.Equal(t, saveResult, string(output)) +} + +func TestIPSetSaveNoMatch(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, ExitCode: 1}, + {Cmd: []string{"grep", "azure-npm-"}}, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + output, err := iMgr.ipsetSave() + require.NoError(t, err) + require.Nil(t, output) } func TestCreateForAllSetTypes(t *testing.T) { + tests := []struct { + name string + withSaveFile bool + }{ + {name: "with save file", withSaveFile: true}, + {name: "no save file", withSaveFile: false}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) + iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) + + var creator *ioutil.FileCreator + if tt.withSaveFile { + creator = iMgr.fileCreatorForApplyWithSaveFile(len(calls), nil) + } else { + creator = iMgr.fileCreatorForApply(len(calls)) + } + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + + expectedLines := []string{ + fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), + fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), + fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), + fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), + fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), + "", + } + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") + }) + } +} + +func TestDestroy(t *testing.T) { + tests := []struct { + name string + withSaveFile bool + }{ + {name: "with save file", withSaveFile: true}, + {name: "no save file", withSaveFile: false}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // without save file + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // remove some members and destroy some sets + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) + require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) + + var creator *ioutil.FileCreator + if tt.withSaveFile { + creator = iMgr.fileCreatorForApplyWithSaveFile(len(calls), nil) + } else { + creator = iMgr.fileCreatorForApply(len(calls)) + } + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + + expectedLines := []string{ + fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), + fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + fmt.Sprintf("-F %s", TestCIDRSet.HashedName), + fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), + fmt.Sprintf("-X %s", TestCIDRSet.HashedName), + fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), + "", + } + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") + }) + } +} + +func TestUpdateWithIdenticalSaveFile(t *testing.T) { calls := []testutils.TestCmd{fakeRestoreSuccessCommand} ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), + fmt.Sprintf("add %s 10.0.0.5", TestKeyPodSet.HashedName), + fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), + fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), + fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), + fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), + fmt.Sprintf(createListFormat, TestKVNSList.HashedName), + fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), + fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) - creator := iMgr.fileCreatorForApply(len(calls), nil) + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) actualLines := testAndSortRestoreFileString(t, creator.ToString()) expectedLines := []string{ @@ -543,16 +752,9 @@ func TestCreateForAllSetTypes(t *testing.T) { fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), - fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), - fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), - fmt.Sprintf("-A %s 10.0.0.5", TestKeyPodSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), - fmt.Sprintf("-A %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), "", } sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) @@ -563,37 +765,67 @@ func TestCreateForAllSetTypes(t *testing.T) { require.False(t, wasFileAltered, "file should not be altered") } -func TestDestroy(t *testing.T) { - // without save file +func TestUpdateWithRealisticSaveFile(t *testing.T) { + // save file doesn't have some sets we're adding and has some sets that: + // - aren't dirty + // - will be deleted + // - have members which we will delete + // - are missing members, which we will add calls := []testutils.TestCmd{fakeRestoreSuccessCommand} ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - // remove some members and destroy some sets + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), // should add 10.0.0.1-5 to this set + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member + fmt.Sprintf("add %s 5.6.7.8", TestNSSet.HashedName), // delete this member + fmt.Sprintf("add %s 5.6.7.9", TestNSSet.HashedName), // delete this member + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // dirty but no member changes in the end + fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), // ignore this set since it's not dirty + fmt.Sprintf("add %s 1.2.3.4", TestKVPodSet.HashedName), // ignore this set since it's not dirty + fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), // should add TestKeyPodSet to this set + fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // keep this member + fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), // delete this member + fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), // ignore this set since it's not dirty + fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), // this set will be deleted + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) - require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.2", "c")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.3", "d")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.4", "e")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.5", "f")) iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) - require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete - iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete + iMgr.CreateIPSets([]*IPSetMetadata{TestKVNSList.Metadata}) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "z")) // set not in save file + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) - creator := iMgr.fileCreatorForApply(len(calls), nil) - actualLines := testAndSortRestoreFileString(t, creator.ToString()) + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + actualLines := testAndSortRestoreFileString(t, creator.ToString()) // adding NSSet and KeyPodSet (should be keeping NSSet and deleting NamedportSet) expectedLines := []string{ fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), - fmt.Sprintf("-A %s 10.0.0.0", TestNSSet.HashedName), - fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), - fmt.Sprintf("-F %s", TestCIDRSet.HashedName), + fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), + fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), + fmt.Sprintf("-A %s 1.2.3.4", TestCIDRSet.HashedName), + fmt.Sprintf("-D %s 5.6.7.8", TestNSSet.HashedName), + fmt.Sprintf("-D %s 5.6.7.9", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.2", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.3", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.4", TestNSSet.HashedName), + fmt.Sprintf("-A %s 10.0.0.5", TestNSSet.HashedName), + fmt.Sprintf("-D %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), + fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), - fmt.Sprintf("-X %s", TestCIDRSet.HashedName), fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), "", } @@ -605,11 +837,319 @@ func TestDestroy(t *testing.T) { require.False(t, wasFileAltered, "file should not be altered") } +func TestHaveTypeProblem(t *testing.T) { + type args struct { + metadata *IPSetMetadata + format string + } + tests := []struct { + name string + args args + wantProblem bool + }{ + { + name: "correct type for nethash", + args: args{ + TestNSSet.Metadata, + createNethashFormat, + }, + wantProblem: false, + }, + { + name: "nethash instead of porthash", + args: args{ + TestNamedportSet.Metadata, + createNethashFormat, + }, + wantProblem: true, + }, + { + name: "nethash instead of list", + args: args{ + TestKeyNSList.Metadata, + createNethashFormat, + }, + wantProblem: true, + }, + { + name: "correct type for porthash", + args: args{ + TestNamedportSet.Metadata, + createPorthashFormat, + }, + wantProblem: false, + }, + { + name: "porthash instead of nethash", + args: args{ + TestNSSet.Metadata, + createPorthashFormat, + }, + wantProblem: true, + }, + { + name: "porthash instead of list", + args: args{ + TestKeyNSList.Metadata, + createPorthashFormat, + }, + wantProblem: true, + }, + { + name: "correct type for list", + args: args{ + TestKeyNSList.Metadata, + createListFormat, + }, + wantProblem: false, + }, + { + name: "list instead of nethash", + args: args{ + TestNSSet.Metadata, + createListFormat, + }, + wantProblem: true, + }, + { + name: "list instead of porthash", + args: args{ + TestNamedportSet.Metadata, + createListFormat, + }, + wantProblem: true, + }, + { + name: "unknown type", + args: args{ + TestKeyNSList.Metadata, + "create %s unknown-type", + }, + wantProblem: true, + }, + { + name: "no rest of line", + args: args{ + TestKeyNSList.Metadata, + "create %s", + }, + wantProblem: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + set := NewIPSet(tt.args.metadata) + line := fmt.Sprintf(tt.args.format, set.HashedName) + splitLine := strings.Split(line, " ") + restOfLine := splitLine[2:] + if tt.wantProblem { + require.True(t, haveTypeProblem(set, restOfLine)) + } else { + require.False(t, haveTypeProblem(set, restOfLine)) + } + }) + } +} + +func TestUpdateWithBadSaveFile(t *testing.T) { + type args struct { + dirtySet []*IPSetMetadata + saveFileLines []string + } + tests := []struct { + name string + args args + expectedLines []string + }{ + { + name: "no create line", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), + fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), + }, + }, + expectedLines: []string{ + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + "", + }, + }, + { + name: "unexpected verb after create", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), + "wrong-verb ...", + }, + }, + expectedLines: []string{ + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + "", + }, + }, + { + name: "non-NPM set", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + "create test-set1 hash:net family inet hashsize 1024 maxelem 65536", + "add test-set1 1.2.3.4", + }, + }, + expectedLines: []string{ + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + "", + }, + }, + { + name: "ignore set we've already parsed", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include + fmt.Sprintf("add %s 4.4.4.4", TestKeyPodSet.HashedName), // include this add (will DELETE this member) + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // ignore this create and ensuing adds since we already included this set + fmt.Sprintf("add %s 5.5.5.5", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) + }, + }, + expectedLines: []string{ + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-D %s 4.4.4.4", TestKeyPodSet.HashedName), + "", + }, + }, + { + name: "set with wrong type", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + fmt.Sprintf(createPorthashFormat, TestKeyPodSet.HashedName), // ignore since wrong type + fmt.Sprintf("add %s 1.2.3.4,tcp", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) + }, + }, + expectedLines: []string{ + // TODO ideally we shouldn't create this set because the line will fail in the first try for ipset restore + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + "", + }, + }, + { + name: "ignore after add with bad parent", + args: args{ + []*IPSetMetadata{TestKeyPodSet.Metadata}, + []string{ + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include this + fmt.Sprintf("add %s 7.7.7.7", TestKeyPodSet.HashedName), // include this add (will DELETE this member) + fmt.Sprintf("add %s 8.8.8.8", TestNSSet.HashedName), // ignore this and jump to next create since it's an unexpected set (will NO-OP [no delete]) + fmt.Sprintf("add %s 9.9.9.9", TestKeyPodSet.HashedName), // ignore add because of error above (will NO-OP [no delete]) + }, + }, + expectedLines: []string{ + fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), + fmt.Sprintf("-D %s 7.7.7.7", TestKeyPodSet.HashedName), + "", + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + saveFileString := strings.Join(tt.args.saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + + iMgr.CreateIPSets(tt.args.dirtySet) + + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + sortedExpectedLines := testAndSortRestoreFileLines(t, tt.expectedLines) + + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") + }) + } +} + func TestFailureOnCreateForNewSet(t *testing.T) { + tests := []struct { + name string + withSaveFile bool + }{ + {name: "with save file", withSaveFile: true}, + {name: "no save file", withSaveFile: false}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order + // test logic: + // - delete a set + // - create three sets, each with two members. the second set to appear will fail to be created + errorLineNum := 2 + setToCreateAlreadyExistsCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), + ExitCode: 1, + } + calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + // add all of these members to the kernel + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member + iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) + + // get original creator and run it the first time + var creator *ioutil.FileCreator + if tt.withSaveFile { + creator = iMgr.fileCreatorForApplyWithSaveFile(len(calls), nil) + } else { + creator = iMgr.fileCreatorForApply(len(calls)) + } + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated + removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run + originalLength := len(expectedLines) + expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") + require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") + }) + } +} + +func TestFailureOnCreateForSetInKernel(t *testing.T) { // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order // test logic: // - delete a set - // - create three sets, each with two members. the second set to appear will fail to be created + // - update three sets already in the kernel, each with a delete and add line. the second set to appear will fail to be created errorLineNum := 2 setToCreateAlreadyExistsCommand := testutils.TestCmd{ Cmd: ipsetRestoreStringSlice, @@ -621,18 +1161,26 @@ func TestFailureOnCreateForNewSet(t *testing.T) { defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete + fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestKeyPodSet.HashedName), // delete + fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestKVPodSet.HashedName), // delete + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + // add all of these members to the kernel - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.4", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "1.2.3.5", "b")) // add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.5", "b")) // add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.4,tcp:567", "a")) // create and add member - require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNamedportSet.Metadata}, "1.2.3.5,tcp:567", "b")) // add member - iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "6.7.8.9", "a")) // add member to kernel + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel + iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) // get original creator and run it the first time - creator := iMgr.fileCreatorForApply(len(calls), nil) + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) originalLines := strings.Split(creator.ToString(), "\n") wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") require.Error(t, err, "ipset restore should fail") @@ -640,11 +1188,70 @@ func TestFailureOnCreateForNewSet(t *testing.T) { // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) - requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKVPodSet.HashedName, TestCIDRSet.HashedName, TestNamedportSet.HashedName}) + requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKeyPodSet.HashedName, TestKVPodSet.HashedName}) expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run originalLength := len(expectedLines) + expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-D") + require.Equal(t, originalLength-1, len(expectedLines), "expected to remove a delete line") expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") - require.Equal(t, originalLength-2, len(expectedLines), "expected to remove two add lines") + require.Equal(t, originalLength-2, len(expectedLines), "expected to remove an add line") + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestFailureOnAddToListInKernel(t *testing.T) { + // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order + // test logic: + // - delete a set + // - update three lists already in the set, each with a delete and add line. the second list to appear will have the failed add + // - create a set and add a member to it + errorLineNum := 8 + memberDoesNotExistCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel + ExitCode: 1, + } + calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + saveFileLines := []string{ + fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), + fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // delete this member + fmt.Sprintf(createListFormat, TestKVNSList.HashedName), + fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestNSSet.HashedName), // delete this member + fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), + fmt.Sprintf("add %s %s", TestNestedLabelList.HashedName, TestNSSet.HashedName), // delete this member + + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // create and add member + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + // rerun the creator after removing previously run lines, and aborting the member-add line that failed + removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) + removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) + require.Equal(t, TestKeyPodSet.HashedName, removedMember) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) actualLines := testAndSortRestoreFileString(t, creator.ToString()) @@ -654,311 +1261,212 @@ func TestFailureOnCreateForNewSet(t *testing.T) { require.False(t, wasFileAltered, "file should not be altered") } -// func TestFailureOnCreateForSetInKernel(t *testing.T) { -// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order -// // test logic: -// // - delete a set -// // - update three sets already in the kernel, each with a delete and add line. the second set to appear will fail to be created -// errorLineNum := 2 -// setToCreateAlreadyExistsCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: Set cannot be created: set with the same name already exists", errorLineNum), -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{setToCreateAlreadyExistsCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestKeyPodSet.HashedName), // delete -// fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestKVPodSet.HashedName), // delete -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// // add all of these members to the kernel -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "6.7.8.9", "a")) // add member to kernel -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKVPodSet.Metadata}, "6.7.8.9", "a")) // add member to kernel -// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyNSList.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestKeyNSList.PrefixName, util.SoftDelete) - -// // get original creator and run it the first time -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// // rerun the creator after removing previously run lines, and aborting the create, adds, and deletes for the second set to updated -// removedSetName := hashedNameOfSetImpacted(t, "-N", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestNSSet.HashedName, TestKeyPodSet.HashedName, TestKVPodSet.HashedName}) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// originalLength := len(expectedLines) -// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-D") -// require.Equal(t, originalLength-1, len(expectedLines), "expected to remove a delete line") -// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-A") -// require.Equal(t, originalLength-2, len(expectedLines), "expected to remove an add line") -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnAddToListInKernel(t *testing.T) { -// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order -// // test logic: -// // - delete a set -// // - update three lists already in the set, each with a delete and add line. the second list to appear will have the failed add -// // - create a set and add a member to it -// errorLineNum := 8 -// memberDoesNotExistCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), -// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // delete this member -// fmt.Sprintf(createListFormat, TestKVNSList.HashedName), -// fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestNSSet.HashedName), // delete this member -// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), -// fmt.Sprintf("add %s %s", TestNestedLabelList.HashedName, TestNSSet.HashedName), // delete this member - -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // create and add member -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestKeyPodSet.Metadata})) // add member to kernel -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// // rerun the creator after removing previously run lines, and aborting the member-add line that failed -// removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) -// removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) -// require.Equal(t, TestKeyPodSet.HashedName, removedMember) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnAddToNewList(t *testing.T) { -// // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order -// // test logic: -// // - delete a set -// // - update a set already in the kernel with a delete and add line -// // - create three lists in the set, each with an add line. the second list to appear will have the failed add -// errorLineNum := 8 -// memberDoesNotExistCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete this member -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "a")) // create and add member -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// // rerun the creator after removing previously run lines, and aborting the member-add line that failed -// removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) -// removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) -// require.Equal(t, TestNSSet.HashedName, removedMember) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnDelete(t *testing.T) { -// // TODO -// } - -// func TestFailureOnFlush(t *testing.T) { -// // test logic: -// // - delete two sets. the first to appear will fail to flush -// // - update a set by deleting a member -// // - create a set with a member -// errorLineNum := 5 -// setDoesNotExistCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: The set with the given name does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{setDoesNotExistCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member -// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet -// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// // rerun the creator after aborting the flush and delete for the set that failed to flush -// removedSetName := hashedNameOfSetImpacted(t, "-F", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// originalLength := len(expectedLines) -// expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-X") -// require.Equal(t, originalLength-1, len(expectedLines), "expected to remove one destroy line") -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnDestroy(t *testing.T) { -// // test logic: -// // - delete two sets. the first to appear will fail to delete -// // - update a set by deleting a member -// // - create a set with a member -// errorLineNum := 7 -// inUseByKernelCommand := testutils.TestCmd{ -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: Set cannot be destroyed: it is in use by a kernel component", errorLineNum), -// ExitCode: 1, -// } -// calls := []testutils.TestCmd{inUseByKernelCommand, fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member -// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet -// iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// originalLines := strings.Split(creator.ToString(), "\n") -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// removedSetName := hashedNameOfSetImpacted(t, "-X", originalLines, errorLineNum) -// requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) -// expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestFailureOnLastLine(t *testing.T) { -// // make sure that the file recovers and returns no error when there are no more lines on the second run -// // test logic: -// // - delete a set -// errorLineNum := 2 -// calls := []testutils.TestCmd{ -// { -// Cmd: ipsetRestoreStringSlice, -// Stdout: fmt.Sprintf("Error in line %d: some destroy error", errorLineNum), -// ExitCode: 1, -// }, -// } -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(2, nil) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.Error(t, err, "ipset restore should fail") -// require.True(t, wasFileAltered, "file should be altered") - -// expectedLines := []string{""} // skip the error line and the lines previously run -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// dptestutils.AssertEqualLines(t, expectedLines, actualLines) -// wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err) -// require.False(t, wasFileAltered, "file should not be altered") -// } +func TestFailureOnAddToNewList(t *testing.T) { + // with respect to the error line, be weary that sets in the save file are processed first and in order, and other sets are processed in random order + // test logic: + // - delete a set + // - update a set already in the kernel with a delete and add line + // - create three lists in the set, each with an add line. the second list to appear will have the failed add + errorLineNum := 8 + memberDoesNotExistCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: Set to be added/deleted/tested as element does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel + ExitCode: 1, + } + calls := []testutils.TestCmd{memberDoesNotExistCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // delete this member + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "a")) // create and add member + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel + require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestNestedLabelList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata})) // add member to kernel + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + // rerun the creator after removing previously run lines, and aborting the member-add line that failed + removedSetName := hashedNameOfSetImpacted(t, "-A", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestKeyNSList.HashedName, TestKVNSList.HashedName, TestNestedLabelList.HashedName}) + removedMember := memberNameOfSetImpacted(t, originalLines, errorLineNum) + require.Equal(t, TestNSSet.HashedName, removedMember) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestFailureOnDelete(t *testing.T) { + // TODO +} + +func TestFailureOnFlush(t *testing.T) { + // test logic: + // - delete two sets. the first to appear will fail to flush + // - update a set by deleting a member + // - create a set with a member + errorLineNum := 5 + setDoesNotExistCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: The set with the given name does not exist", errorLineNum), // this error might happen if the cache is out of date with the kernel + ExitCode: 1, + } + calls := []testutils.TestCmd{setDoesNotExistCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member + fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + // rerun the creator after aborting the flush and delete for the set that failed to flush + removedSetName := hashedNameOfSetImpacted(t, "-F", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run + originalLength := len(expectedLines) + expectedLines = removeOperationsForSet(expectedLines, removedSetName, "-X") + require.Equal(t, originalLength-1, len(expectedLines), "expected to remove one destroy line") + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestFailureOnDestroy(t *testing.T) { + // test logic: + // - delete two sets. the first to appear will fail to delete + // - update a set by deleting a member + // - create a set with a member + errorLineNum := 7 + inUseByKernelCommand := testutils.TestCmd{ + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: Set cannot be destroyed: it is in use by a kernel component", errorLineNum), + ExitCode: 1, + } + calls := []testutils.TestCmd{inUseByKernelCommand, fakeRestoreSuccessCommand} + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + saveFileLines := []string{ + fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), + fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member + fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), // delete this member + } + saveFileString := strings.Join(saveFileLines, "\n") + saveFileBytes := []byte(saveFileString) + + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) + originalLines := strings.Split(creator.ToString(), "\n") + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + removedSetName := hashedNameOfSetImpacted(t, "-X", originalLines, errorLineNum) + requireStringInSlice(t, removedSetName, []string{TestKVPodSet.HashedName, TestCIDRSet.HashedName}) + expectedLines := originalLines[errorLineNum:] // skip the error line and the lines previously run + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") +} + +func TestFailureOnLastLine(t *testing.T) { + tests := []struct { + name string + withSaveFile bool + }{ + {name: "with save file", withSaveFile: true}, + {name: "no save file", withSaveFile: false}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // make sure that the file recovers and returns no error when there are no more lines on the second run + // test logic: + // - delete a set + errorLineNum := 2 + calls := []testutils.TestCmd{ + { + Cmd: ipsetRestoreStringSlice, + Stdout: fmt.Sprintf("Error in line %d: some destroy error", errorLineNum), + ExitCode: 1, + }, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + var creator *ioutil.FileCreator + if tt.withSaveFile { + creator = iMgr.fileCreatorForApplyWithSaveFile(2, nil) + } else { + creator = iMgr.fileCreatorForApply(2) + } + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.Error(t, err, "ipset restore should fail") + require.True(t, wasFileAltered, "file should be altered") + + expectedLines := []string{""} // skip the error line and the lines previously run + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, expectedLines, actualLines) + wasFileAltered, err = creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err) + require.False(t, wasFileAltered, "file should not be altered") + }) + } +} func testAndSortRestoreFileString(t *testing.T, multilineString string) []string { return testAndSortRestoreFileLines(t, strings.Split(multilineString, "\n")) @@ -1059,418 +1567,3 @@ func removeOperationsForSet(lines []string, hashedSetName, operation string) []s } return goodLines } - -// func TestNextCreateLine(t *testing.T) { -// createLine := "create test-list1 list:set size 8" -// addLine := "add test-set1 1.2.3.4" -// createLineWithNewline := createLine + "\n" -// addLineWithNewline := addLine + "\n" -// tests := []struct { -// name string -// lines []string -// expectedReadIndex int -// expectedLine []byte -// }{ -// // parse.Line will omit the newline at the end of the line unless it's the last line -// { -// name: "empty save file", -// lines: []string{}, -// expectedReadIndex: 0, -// expectedLine: nil, -// }, -// { -// name: "no creates", -// lines: []string{addLineWithNewline}, -// expectedReadIndex: len(addLineWithNewline), -// expectedLine: []byte(addLineWithNewline), -// }, -// { -// name: "start with create", -// lines: []string{createLine, addLineWithNewline}, -// expectedReadIndex: len(createLineWithNewline), -// expectedLine: []byte(createLine), -// }, -// { -// name: "create after adds", -// lines: []string{addLine, addLine, createLineWithNewline}, -// expectedReadIndex: 2*len(addLine+"\n") + len(createLine+"\n"), -// expectedLine: []byte(createLineWithNewline), -// }, -// } -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// saveFile := []byte(strings.Join(tt.lines, "\n")) -// line, readIndex := nextCreateLine(0, saveFile) -// require.Equal(t, tt.expectedReadIndex, readIndex) -// require.Equal(t, tt.expectedLine, line) -// }) -// } -// // fmt.Println(string([]byte(addLine + addLine, createLineWithNewline})[:78])) -// } - -// func TestHaveTypeProblem(t *testing.T) { -// type args struct { -// metadata *IPSetMetadata -// format string -// } -// tests := []struct { -// name string -// args args -// wantProblem bool -// }{ -// { -// name: "correct type for nethash", -// args: args{ -// TestNSSet.Metadata, -// createNethashFormat, -// }, -// wantProblem: false, -// }, -// { -// name: "nethash instead of porthash", -// args: args{ -// TestNamedportSet.Metadata, -// createNethashFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "nethash instead of list", -// args: args{ -// TestKeyNSList.Metadata, -// createNethashFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "correct type for porthash", -// args: args{ -// TestNamedportSet.Metadata, -// createPorthashFormat, -// }, -// wantProblem: false, -// }, -// { -// name: "porthash instead of nethash", -// args: args{ -// TestNSSet.Metadata, -// createPorthashFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "porthash instead of list", -// args: args{ -// TestKeyNSList.Metadata, -// createPorthashFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "correct type for list", -// args: args{ -// TestKeyNSList.Metadata, -// createListFormat, -// }, -// wantProblem: false, -// }, -// { -// name: "list instead of nethash", -// args: args{ -// TestNSSet.Metadata, -// createListFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "list instead of porthash", -// args: args{ -// TestNamedportSet.Metadata, -// createListFormat, -// }, -// wantProblem: true, -// }, -// { -// name: "unknown type", -// args: args{ -// TestKeyNSList.Metadata, -// "create %s unknown-type", -// }, -// wantProblem: true, -// }, -// { -// name: "no rest of line", -// args: args{ -// TestKeyNSList.Metadata, -// "create %s", -// }, -// wantProblem: true, -// }, -// } -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// set := NewIPSet(tt.args.metadata) -// line := fmt.Sprintf(tt.args.format, set.HashedName) -// splitLine := strings.Split(line, " ") -// restOfLine := splitLine[2:] -// if tt.wantProblem { -// require.True(t, haveTypeProblem(set, restOfLine)) -// } else { -// require.False(t, haveTypeProblem(set, restOfLine)) -// } -// }) -// } -// } - -// func TestUpdateWithIdenticalSaveFile(t *testing.T) { -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.1", TestNSSet.HashedName), -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), -// fmt.Sprintf("add %s 10.0.0.5", TestKeyPodSet.HashedName), -// fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), -// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), -// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), -// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), -// fmt.Sprintf(createListFormat, TestKVNSList.HashedName), -// fmt.Sprintf("add %s %s", TestKVNSList.HashedName, TestKVPodSet.HashedName), -// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.5", "c")) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNamedportSet.Metadata}) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKVNSList.Metadata}, []*IPSetMetadata{TestKVPodSet.Metadata})) -// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) - -// expectedLines := []string{ -// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKVPodSet.HashedName), -// fmt.Sprintf("-N %s --exist hash:ip,port", TestNamedportSet.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestNestedLabelList.HashedName), -// "", -// } -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err, "ipset restore should be successful") -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestUpdateWithRealisticSaveFile(t *testing.T) { -// // save file doesn't have some sets we're adding and has some sets that: -// // - aren't dirty -// // - will be deleted -// // - have members which we will delete -// // - are missing members, which we will add -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileLines := []string{ -// fmt.Sprintf(createNethashFormat, TestNSSet.HashedName), // should add 10.0.0.1-5 to this set -// fmt.Sprintf("add %s 10.0.0.0", TestNSSet.HashedName), // keep this member -// fmt.Sprintf("add %s 5.6.7.8", TestNSSet.HashedName), // delete this member -// fmt.Sprintf("add %s 5.6.7.9", TestNSSet.HashedName), // delete this member -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // dirty but no member changes in the end -// fmt.Sprintf(createNethashFormat, TestKVPodSet.HashedName), // ignore this set since it's not dirty -// fmt.Sprintf("add %s 1.2.3.4", TestKVPodSet.HashedName), // ignore this set since it's not dirty -// fmt.Sprintf(createListFormat, TestKeyNSList.HashedName), // should add TestKeyPodSet to this set -// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNSSet.HashedName), // keep this member -// fmt.Sprintf("add %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), // delete this member -// fmt.Sprintf(createPorthashFormat, TestNamedportSet.HashedName), // ignore this set since it's not dirty -// fmt.Sprintf(createListFormat, TestNestedLabelList.HashedName), // this set will be deleted -// } -// saveFileString := strings.Join(saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.2", "c")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.3", "d")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.4", "e")) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.5", "f")) -// iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) -// require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) -// iMgr.CreateIPSets([]*IPSetMetadata{TestKVNSList.Metadata}) -// require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "z")) // set not in save file -// iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete -// iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) // adding NSSet and KeyPodSet (should be keeping NSSet and deleting NamedportSet) - -// expectedLines := []string{ -// fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKeyNSList.HashedName), -// fmt.Sprintf("-N %s --exist setlist", TestKVNSList.HashedName), -// fmt.Sprintf("-N %s --exist nethash maxelem 4294967295", TestCIDRSet.HashedName), -// fmt.Sprintf("-A %s 1.2.3.4", TestCIDRSet.HashedName), -// fmt.Sprintf("-D %s 5.6.7.8", TestNSSet.HashedName), -// fmt.Sprintf("-D %s 5.6.7.9", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.1", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.2", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.3", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.4", TestNSSet.HashedName), -// fmt.Sprintf("-A %s 10.0.0.5", TestNSSet.HashedName), -// fmt.Sprintf("-D %s %s", TestKeyNSList.HashedName, TestNamedportSet.HashedName), -// fmt.Sprintf("-A %s %s", TestKeyNSList.HashedName, TestKeyPodSet.HashedName), -// fmt.Sprintf("-F %s", TestNestedLabelList.HashedName), -// fmt.Sprintf("-X %s", TestNestedLabelList.HashedName), -// "", -// } -// sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) - -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err, "ipset restore should be successful") -// require.False(t, wasFileAltered, "file should not be altered") -// } - -// func TestUpdateWithBadSaveFile(t *testing.T) { -// type args struct { -// dirtySet []*IPSetMetadata -// saveFileLines []string -// } -// tests := []struct { -// name string -// args args -// expectedLines []string -// }{ -// { -// name: "no create line", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), -// fmt.Sprintf("add %s 1.1.1.1", TestKeyPodSet.HashedName), -// }, -// }, -// expectedLines: []string{ -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// { -// name: "unexpected verb after create", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), -// "wrong-verb ...", -// }, -// }, -// expectedLines: []string{ -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// { -// name: "non-NPM set", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// "create test-set1 hash:net family inet hashsize 1024 maxelem 65536", -// "add test-set1 1.2.3.4", -// }, -// }, -// expectedLines: []string{ -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// { -// name: "ignore set we've already parsed", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include -// fmt.Sprintf("add %s 4.4.4.4", TestKeyPodSet.HashedName), // include this add (will DELETE this member) -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // ignore this create and ensuing adds since we already included this set -// fmt.Sprintf("add %s 5.5.5.5", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) -// }, -// }, -// expectedLines: []string{ -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-D %s 4.4.4.4", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// { -// name: "set with wrong type", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// fmt.Sprintf(createPorthashFormat, TestKeyPodSet.HashedName), // ignore since wrong type -// fmt.Sprintf("add %s 1.2.3.4,tcp", TestKeyPodSet.HashedName), // ignore this add (will NO-OP [no delete]) -// }, -// }, -// expectedLines: []string{ -// // TODO ideally we shouldn't create this set because the line will fail in the first try for ipset restore -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// { -// name: "ignore after add with bad parent", -// args: args{ -// []*IPSetMetadata{TestKeyPodSet.Metadata}, -// []string{ -// fmt.Sprintf(createNethashFormat, TestKeyPodSet.HashedName), // include this -// fmt.Sprintf("add %s 7.7.7.7", TestKeyPodSet.HashedName), // include this add (will DELETE this member) -// fmt.Sprintf("add %s 8.8.8.8", TestNSSet.HashedName), // ignore this and jump to next create since it's an unexpected set (will NO-OP [no delete]) -// fmt.Sprintf("add %s 9.9.9.9", TestKeyPodSet.HashedName), // ignore add because of error above (will NO-OP [no delete]) -// }, -// }, -// expectedLines: []string{ -// fmt.Sprintf("-N %s --exist nethash", TestKeyPodSet.HashedName), -// fmt.Sprintf("-D %s 7.7.7.7", TestKeyPodSet.HashedName), -// "", -// }, -// }, -// } -// for _, tt := range tests { -// tt := tt -// t.Run(tt.name, func(t *testing.T) { -// calls := []testutils.TestCmd{fakeRestoreSuccessCommand} -// ioshim := common.NewMockIOShim(calls) -// defer ioshim.VerifyCalls(t, calls) -// iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) - -// saveFileString := strings.Join(tt.args.saveFileLines, "\n") -// saveFileBytes := []byte(saveFileString) - -// iMgr.CreateIPSets(tt.args.dirtySet) - -// creator := iMgr.fileCreatorForApply(len(calls), saveFileBytes) -// actualLines := testAndSortRestoreFileString(t, creator.ToString()) -// sortedExpectedLines := testAndSortRestoreFileLines(t, tt.expectedLines) - -// dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) -// wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") -// require.NoError(t, err, "ipset restore should be successful") -// require.False(t, wasFileAltered, "file should not be altered") -// }) -// } -// } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index cc8a8b229b..0c00b30323 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -163,7 +163,7 @@ func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkP // for faster look up changing a slice to map existingSetNames := make(map[string]struct{}) - for _, setName := range existingSets { + for setName := range existingSets { existingSetNames[setName] = struct{}{} } // (TODO) remove this log line later From e1c541a3898f52e122d94fa1c5a341e703355039 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 11 Apr 2022 16:40:06 -0700 Subject: [PATCH 11/17] fix windows --- npm/pkg/dataplane/ipsets/dirtycache.go | 2 ++ npm/pkg/dataplane/ipsets/dirtycache_linux.go | 5 +++++ npm/pkg/dataplane/ipsets/dirtycache_test.go | 21 +++++++++++++++++++ .../dataplane/ipsets/dirtycache_windows.go | 16 ++++++++------ .../dataplane/ipsets/ipsetmanager_windows.go | 7 +++---- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index 5ffc1c9dcc..e2fa1230e9 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -11,6 +11,8 @@ package ipsets type dirtyCacheInterface interface { // reset empties dirty cache reset() + // resetAddOrUpdateCache empties the dirty cache of sets to be created or updated + resetAddOrUpdateCache() // create will mark the new set to be created. create(set *IPSet) // addMember will mark the set to be updated and track the member to be added (if implemented). diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index 1599f6ca26..e4312159a5 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -39,6 +39,11 @@ func (dc *dirtyCache) reset() { dc.toDestroyCache = make(map[string]*memberDiff) } +func (dc *dirtyCache) resetAddOrUpdateCache() { + dc.toCreateCache = make(map[string]*memberDiff) + dc.toUpdateCache = make(map[string]*memberDiff) +} + func (dc *dirtyCache) create(set *IPSet) { // error checking if _, ok := dc.toCreateCache[set.Name]; ok { diff --git a/npm/pkg/dataplane/ipsets/dirtycache_test.go b/npm/pkg/dataplane/ipsets/dirtycache_test.go index aa7acabf19..af5999ac6f 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_test.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_test.go @@ -39,6 +39,27 @@ func TestDirtyCacheReset(t *testing.T) { assertDirtyCache(t, dc, &dirtyCacheResults{}) } +func TestDirtyCacheResetAddOrUpdate(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + set2 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + set3 := NewIPSet(NewIPSetMetadata("set3", Namespace)) + set4 := NewIPSet(NewIPSetMetadata("set4", Namespace)) + dc := newDirtyCache() + dc.create(set1) + dc.addMember(set2, ip) + dc.deleteMember(set3, "4.4.4.4") + set4.IPPodKey[ip] = podKey + dc.destroy(set4) + dc.resetAddOrUpdateCache() + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set4.Name: { + toDelete: []string{ip}, + }, + }, + }) +} + func TestDirtyCacheCreate(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index d9d0226618..854779037e 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -18,6 +18,10 @@ func (dc *dirtyCache) reset() { dc.toDeleteCache = make(map[string]struct{}) } +func (dc *dirtyCache) resetAddOrUpdateCache() { + dc.toAddOrUpdateCache = make(map[string]struct{}) +} + func (dc *dirtyCache) create(set *IPSet) { putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) } @@ -42,19 +46,19 @@ func putInCacheAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]st } func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { - result := make([]string, 0, len(dc.toAddOrUpdateCache)) + m := make(map[string]struct{}, 0, len(dc.toAddOrUpdateCache)) for setName := range dc.toAddOrUpdateCache { - result = append(result, setName) + m[setName] = struct{}{} } - return result + return m } func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { - result := make([]string, 0, len(dc.toDeleteCache)) + m := make(map[string]struct{}, 0, len(dc.toDeleteCache)) for setName := range dc.toDeleteCache { - result = append(result, setName) + m[setName] = struct{}{} } - return result + return m } func (dc *dirtyCache) numSetsToAddOrUpdate() int { diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index 0c00b30323..d2587196f8 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -124,8 +124,7 @@ func (iMgr *IPSetManager) applyIPSets() error { } } - // FIXME uncomment and handle? - // iMgr.toAddOrUpdateCache = make(map[string]struct{}) + dc.resetAddOrUpdateCache() if len(setPolicyBuilder.toDeleteSets) > 0 { err = iMgr.modifySetPolicies(network, hcn.RequestTypeRemove, setPolicyBuilder.toDeleteSets) @@ -163,13 +162,13 @@ func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkP // for faster look up changing a slice to map existingSetNames := make(map[string]struct{}) - for setName := range existingSets { + for _, setName := range existingSets { existingSetNames[setName] = struct{}{} } // (TODO) remove this log line later klog.Infof("toAddUpdateSetNames %+v \n ", toAddUpdateSetNames) klog.Infof("existingSetNames %+v \n ", existingSetNames) - for _, setName := range toAddUpdateSetNames { + for setName := range toAddUpdateSetNames { set, exists := iMgr.setMap[setName] // check if the Set exists if !exists { return nil, errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("ipset %s does not exist", setName)) From a1b0a7772449130c04692a2024aa5282ffc9dc30 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 11 Apr 2022 16:55:38 -0700 Subject: [PATCH 12/17] fix windows 2 --- npm/pkg/dataplane/ipsets/dirtycache_windows.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 854779037e..a0fbc4476f 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -7,7 +7,7 @@ type dirtyCache struct { toDeleteCache map[string]struct{} } -func newDirtyCache() dirtyCacheMaintainer { +func newDirtyCache() dirtyCacheInterface { dc := &dirtyCache{} dc.reset() return dc @@ -46,7 +46,7 @@ func putInCacheAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]st } func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { - m := make(map[string]struct{}, 0, len(dc.toAddOrUpdateCache)) + m := make(map[string]struct{}, len(dc.toAddOrUpdateCache)) for setName := range dc.toAddOrUpdateCache { m[setName] = struct{}{} } @@ -54,7 +54,7 @@ func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { } func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { - m := make(map[string]struct{}, 0, len(dc.toDeleteCache)) + m := make(map[string]struct{}, len(dc.toDeleteCache)) for setName := range dc.toDeleteCache { m[setName] = struct{}{} } From 6e016cfb5eeec007391b4e5fc0109f01a07a23fe Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Mon, 11 Apr 2022 17:11:16 -0700 Subject: [PATCH 13/17] fix windows 3 --- npm/pkg/dataplane/ipsets/ipsetmanager_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index d2587196f8..cfd3b5e74e 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -124,7 +124,7 @@ func (iMgr *IPSetManager) applyIPSets() error { } } - dc.resetAddOrUpdateCache() + iMgr.dirtyCache.resetAddOrUpdateCache() if len(setPolicyBuilder.toDeleteSets) > 0 { err = iMgr.modifySetPolicies(network, hcn.RequestTypeRemove, setPolicyBuilder.toDeleteSets) From dd51ede08d90ae895f77b2c5c855777bc81840e6 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 13 Apr 2022 10:54:24 -0700 Subject: [PATCH 14/17] UTs and dirty cache in same struct shared between each OS --- npm/pkg/dataplane/ipsets/dirtycache.go | 238 ++++++++- npm/pkg/dataplane/ipsets/dirtycache_linux.go | 222 +------- .../dataplane/ipsets/dirtycache_linux_test.go | 28 + npm/pkg/dataplane/ipsets/dirtycache_test.go | 501 ++++++++++++------ .../dataplane/ipsets/dirtycache_windows.go | 93 +--- .../ipsets/dirtycache_windows_test.go | 7 + npm/pkg/dataplane/ipsets/ipsetmanager.go | 4 +- .../dataplane/ipsets/ipsetmanager_linux.go | 15 +- .../ipsets/ipsetmanager_linux_test.go | 110 +++- npm/pkg/dataplane/ipsets/ipsetmanager_test.go | 51 +- .../dataplane/ipsets/ipsetmanager_windows.go | 2 +- 11 files changed, 782 insertions(+), 489 deletions(-) create mode 100644 npm/pkg/dataplane/ipsets/dirtycache_linux_test.go create mode 100644 npm/pkg/dataplane/ipsets/dirtycache_windows_test.go diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index e2fa1230e9..d0f125c7c3 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -1,12 +1,30 @@ package ipsets +import ( + "fmt" + "strings" + + "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/util" + "k8s.io/klog" +) + /* dirtyCacheInterface will maintain the dirty cache. It may maintain membersToAdd and membersToDelete. Members are either IPs, CIDRs, IP-Port pairs, or prefixed set names if the parent is a list. Assumptions: - FIXME: TODO + - if the set becomes dirty via update or destroy, then the set WAS in the kernel before + - update won't be called unless the set IS in the kernel OR create was called before + - destroy won't be called unless the set IS in the kernel OR create was called before + - if the set becomes dirty via create, then the set was NOT in the kernel before + - create won't be called twice (unless a destroy call comes in between) + + Examples of Expected Behavior: + - if a set is created and then destroyed, that set will not be in the dirty cache anymore + - if a set is updated and then destroyed, that set will be in the delete cache + - if a the only operations on a set are adding and removing the same member, the set may still be in the dirty cache, but the member will be untracked */ type dirtyCacheInterface interface { // reset empties dirty cache @@ -21,10 +39,10 @@ type dirtyCacheInterface interface { deleteMember(set *IPSet, member string) // delete will mark the set to be deleted in the cache destroy(set *IPSet) - // getSetsToAddOrUpdate returns the set names to be added or updated - getSetsToAddOrUpdate() map[string]struct{} - // getSetsToDelete returns the set names to be deleted - getSetsToDelete() map[string]struct{} + // setsToAddOrUpdate returns the set names to be added or updated + setsToAddOrUpdate() map[string]struct{} + // setsToDelete returns the set names to be deleted + setsToDelete() map[string]struct{} // numSetsToAddOrUpdate returns the number of sets to be added or updated numSetsToAddOrUpdate() int // numSetsToDelete returns the number of sets to be deleted @@ -37,8 +55,210 @@ type dirtyCacheInterface interface { printAddOrUpdateCache() string // printDeleteCache returns a string representation of the delete cache printDeleteCache() string - // getOriginalMembers returns the members which should be added for the set. - getMembersToAdd(setName string) map[string]struct{} - // getOriginalMembers returns the members which should be deleted for the set. - getMembersToDelete(setName string) map[string]struct{} + // memberDiff returns the member diff for the set. + // Will create a new memberDiff if the setName isn't in the dirty cache. + memberDiff(setName string) *memberDiff +} + +type dirtyCache struct { + // all maps have keys of set names and values of members to add/delete + toCreateCache map[string]*memberDiff + toUpdateCache map[string]*memberDiff + toDestroyCache map[string]*memberDiff +} + +func newDirtyCache() *dirtyCache { + dc := &dirtyCache{} + dc.reset() + return dc +} + +func (dc *dirtyCache) reset() { + dc.toCreateCache = make(map[string]*memberDiff) + dc.toUpdateCache = make(map[string]*memberDiff) + dc.toDestroyCache = make(map[string]*memberDiff) +} + +func (dc *dirtyCache) resetAddOrUpdateCache() { + dc.toCreateCache = make(map[string]*memberDiff) + dc.toUpdateCache = make(map[string]*memberDiff) +} + +func (dc *dirtyCache) create(set *IPSet) { + // error checking + if _, ok := dc.toCreateCache[set.Name]; ok { + msg := fmt.Sprintf("create: set %s should not already be in the toCreateCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return + } + if _, ok := dc.toUpdateCache[set.Name]; ok { + msg := fmt.Sprintf("create: set %s should not be in the toUpdateCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return + } + + diff, ok := dc.toDestroyCache[set.Name] + if ok { + // transfer from toDestroyCache to toUpdateCache and maintain member diff + dc.toUpdateCache[set.Name] = diff + delete(dc.toDestroyCache, set.Name) + } else { + // put in the toCreateCache + dc.toCreateCache[set.Name] = diffOnCreate(set) + } +} + +// could optimize Linux to remove from toUpdateCache if there were no member diff afterwards, +// but leaving as is prevents difference between OS caches +func (dc *dirtyCache) addMember(set *IPSet, member string) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("addMember: set %s should not be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return + } + + diff, ok := dc.toCreateCache[set.Name] + if !ok { + diff, ok = dc.toUpdateCache[set.Name] + if !ok { + diff = newMemberDiff() + dc.toUpdateCache[set.Name] = diff + } + } + diff.addMember(member) +} + +// could optimize Linux to remove from toUpdateCache if there were no member diff afterwards, +// but leaving as is prevents difference between OS caches +func (dc *dirtyCache) deleteMember(set *IPSet, member string) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("deleteMember: set %s should not be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return + } + + diff, ok := dc.toCreateCache[set.Name] + if ok { + // don't mark a member to be deleted if it never existed in the kernel + diff.deleteMemberIfExists(member) + } else { + diff, ok = dc.toUpdateCache[set.Name] + if !ok { + diff = newMemberDiff() + dc.toUpdateCache[set.Name] = diff + } + diff.deleteMember(member) + } +} + +func (dc *dirtyCache) destroy(set *IPSet) { + // error checking + if dc.isSetToDelete(set.Name) { + msg := fmt.Sprintf("destroy: set %s should not already be in the toDestroyCache", set.Name) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return + } + + if _, ok := dc.toCreateCache[set.Name]; !ok { + // mark all current members as membersToDelete to accommodate force delete + diff, ok := dc.toUpdateCache[set.Name] + if !ok { + diff = newMemberDiff() + } + if set.Kind == HashSet { + for ip := range set.IPPodKey { + diff.deleteMember(ip) + } + } else { + for _, memberSet := range set.MemberIPSets { + diff.deleteMember(memberSet.HashedName) + } + } + // must call this after deleteMember for correct member diff + diff.resetMembersToAdd() + + // put the set/diff in the toDestroyCache + dc.toDestroyCache[set.Name] = diff + } + // remove set from toCreateCache or toUpdateCache if necessary + // if the set/diff was in the toCreateCache before, we'll forget about it + delete(dc.toCreateCache, set.Name) + delete(dc.toUpdateCache, set.Name) +} + +func (dc *dirtyCache) setsToAddOrUpdate() map[string]struct{} { + sets := make(map[string]struct{}, len(dc.toCreateCache)+len(dc.toUpdateCache)) + for set := range dc.toCreateCache { + sets[set] = struct{}{} + } + for set := range dc.toUpdateCache { + sets[set] = struct{}{} + } + return sets +} + +func (dc *dirtyCache) setsToDelete() map[string]struct{} { + sets := make(map[string]struct{}, len(dc.toDestroyCache)) + for setName := range dc.toDestroyCache { + sets[setName] = struct{}{} + } + return sets +} + +func (dc *dirtyCache) numSetsToAddOrUpdate() int { + return len(dc.toCreateCache) + len(dc.toUpdateCache) +} + +func (dc *dirtyCache) numSetsToDelete() int { + return len(dc.toDestroyCache) +} + +func (dc *dirtyCache) isSetToAddOrUpdate(setName string) bool { + _, ok1 := dc.toCreateCache[setName] + _, ok2 := dc.toUpdateCache[setName] + return ok1 || ok2 +} + +func (dc *dirtyCache) isSetToDelete(setName string) bool { + _, ok := dc.toDestroyCache[setName] + return ok +} + +func (dc *dirtyCache) printAddOrUpdateCache() string { + toCreate := make([]string, 0, len(dc.toCreateCache)) + for setName, diff := range dc.toCreateCache { + toCreate = append(toCreate, fmt.Sprintf("%s: %+v", setName, diff)) + } + toUpdate := make([]string, 0, len(dc.toUpdateCache)) + for setName, diff := range dc.toUpdateCache { + toUpdate = append(toUpdate, fmt.Sprintf("%s: %+v", setName, diff)) + } + return fmt.Sprintf("[to create: %+v], [to update: %+v]", strings.Join(toCreate, ","), strings.Join(toUpdate, ",")) +} + +func (dc *dirtyCache) printDeleteCache() string { + return fmt.Sprintf("%+v", dc.toDestroyCache) +} + +func (dc *dirtyCache) memberDiff(setName string) *memberDiff { + diff, ok := dc.toCreateCache[setName] + if ok { + return diff + } + diff, ok = dc.toUpdateCache[setName] + if ok { + return diff + } + diff, ok = dc.toDestroyCache[setName] + if ok { + return diff + } + return newMemberDiff() } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index e4312159a5..e3ba14f90c 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -1,20 +1,5 @@ package ipsets -import ( - "fmt" - - "github.com/Azure/azure-container-networking/npm/metrics" - "github.com/Azure/azure-container-networking/npm/util" - "k8s.io/klog" -) - -type dirtyCache struct { - // all maps have keys of set names and values of members to add/delete - toCreateCache map[string]*memberDiff - toUpdateCache map[string]*memberDiff - toDestroyCache map[string]*memberDiff -} - type memberDiff struct { membersToAdd map[string]struct{} membersToDelete map[string]struct{} @@ -27,71 +12,27 @@ func newMemberDiff() *memberDiff { } } -func newDirtyCache() dirtyCacheInterface { - dc := &dirtyCache{} - dc.reset() - return dc -} - -func (dc *dirtyCache) reset() { - dc.toCreateCache = make(map[string]*memberDiff) - dc.toUpdateCache = make(map[string]*memberDiff) - dc.toDestroyCache = make(map[string]*memberDiff) -} - -func (dc *dirtyCache) resetAddOrUpdateCache() { - dc.toCreateCache = make(map[string]*memberDiff) - dc.toUpdateCache = make(map[string]*memberDiff) -} - -func (dc *dirtyCache) create(set *IPSet) { - // error checking - if _, ok := dc.toCreateCache[set.Name]; ok { - msg := fmt.Sprintf("create: set %s should not already be in the toCreateCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) - } - if _, ok := dc.toUpdateCache[set.Name]; ok { - msg := fmt.Sprintf("create: set %s should not be in the toUpdateCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) - } - - diff, ok := dc.toDestroyCache[set.Name] - if ok { - // transfer from toDestroyCache to toUpdateCache and maintain member diff - dc.toUpdateCache[set.Name] = diff - delete(dc.toDestroyCache, set.Name) - } else { - // put in the toCreateCache and mark all current members as membersToAdd - var members map[string]struct{} - if set.Kind == HashSet { - members = make(map[string]struct{}, len(set.IPPodKey)) - for ip := range set.IPPodKey { - members[ip] = struct{}{} - } - } else { - members = make(map[string]struct{}, len(set.MemberIPSets)) - for _, memberSet := range set.MemberIPSets { - members[memberSet.HashedName] = struct{}{} - } +func diffOnCreate(set *IPSet) *memberDiff { + // mark all current members as membersToAdd + var members map[string]struct{} + if set.Kind == HashSet { + members = make(map[string]struct{}, len(set.IPPodKey)) + for ip := range set.IPPodKey { + members[ip] = struct{}{} } - dc.toCreateCache[set.Name] = &memberDiff{ - membersToAdd: members, + } else { + members = make(map[string]struct{}, len(set.MemberIPSets)) + for _, memberSet := range set.MemberIPSets { + members[memberSet.HashedName] = struct{}{} } } - fmt.Println("here") -} - -func (dc *dirtyCache) addMember(set *IPSet, member string) { - // error checking - if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("addMember: set %s should not be in the toDestroyCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) + return &memberDiff{ + membersToAdd: members, + membersToDelete: make(map[string]struct{}), } +} - diff := dc.getCreateOrUpdateDiff(set) +func (diff *memberDiff) addMember(member string) { _, ok := diff.membersToDelete[member] if ok { delete(diff.membersToDelete, member) @@ -100,15 +41,7 @@ func (dc *dirtyCache) addMember(set *IPSet, member string) { } } -func (dc *dirtyCache) deleteMember(set *IPSet, member string) { - // error checking - if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("deleteMember: set %s should not be in the toDestroyCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) - } - - diff := dc.getCreateOrUpdateDiff(set) +func (diff *memberDiff) deleteMember(member string) { _, ok := diff.membersToAdd[member] if ok { delete(diff.membersToAdd, member) @@ -117,123 +50,10 @@ func (dc *dirtyCache) deleteMember(set *IPSet, member string) { } } -func (dc *dirtyCache) getCreateOrUpdateDiff(set *IPSet) *memberDiff { - diff, ok := dc.toCreateCache[set.Name] - if !ok { - diff, ok = dc.toUpdateCache[set.Name] - if !ok { - diff = newMemberDiff() - dc.toUpdateCache[set.Name] = diff - } - } - return diff -} - -func (dc *dirtyCache) destroy(set *IPSet) { - // error checking - if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("destroy: set %s should not already be in the toDestroyCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) - } - - if _, ok := dc.toCreateCache[set.Name]; !ok { - // modify the diff in the toUpdateCache - // mark all current members as membersToDelete to accommodate force delete - if set.Kind == HashSet { - for ip := range set.IPPodKey { - dc.deleteMember(set, ip) - } - } else { - for _, memberSet := range set.MemberIPSets { - dc.deleteMember(set, memberSet.HashedName) - } - } - } - // put the diff in the toDestroyCache - diff := dc.getCreateOrUpdateDiff(set) - dc.toDestroyCache[set.Name] = diff - delete(dc.toCreateCache, set.Name) - delete(dc.toUpdateCache, set.Name) -} - -func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { - sets := make(map[string]struct{}, len(dc.toCreateCache)+len(dc.toUpdateCache)) - for set := range dc.toCreateCache { - sets[set] = struct{}{} - } - for set := range dc.toUpdateCache { - sets[set] = struct{}{} - } - return sets -} - -func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { - sets := make(map[string]struct{}, len(dc.toDestroyCache)) - for setName := range dc.toDestroyCache { - sets[setName] = struct{}{} - } - return sets -} - -func (dc *dirtyCache) numSetsToAddOrUpdate() int { - return len(dc.toCreateCache) + len(dc.toUpdateCache) -} - -func (dc *dirtyCache) numSetsToDelete() int { - return len(dc.toDestroyCache) -} - -func (dc *dirtyCache) isSetToAddOrUpdate(setName string) bool { - _, ok1 := dc.toCreateCache[setName] - _, ok2 := dc.toUpdateCache[setName] - return ok1 || ok2 -} - -func (dc *dirtyCache) isSetToDelete(setName string) bool { - _, ok := dc.toDestroyCache[setName] - return ok -} - -func (dc *dirtyCache) printAddOrUpdateCache() string { - return fmt.Sprintf("[to create: %+v], [to update: %+v]", dc.toCreateCache, dc.toUpdateCache) +func (diff *memberDiff) deleteMemberIfExists(member string) { + delete(diff.membersToAdd, member) } -func (dc *dirtyCache) printDeleteCache() string { - return fmt.Sprintf("%+v", dc.toDestroyCache) -} - -func (dc *dirtyCache) getMembersToAdd(setName string) map[string]struct{} { - fmt.Println("hey1") - diff, ok := dc.toCreateCache[setName] - if ok { - return diff.membersToAdd - } - fmt.Println("hey2") - diff, ok = dc.toUpdateCache[setName] - if ok { - return diff.membersToAdd - } - fmt.Println("hey3") - diff, ok = dc.toDestroyCache[setName] - if ok { - return diff.membersToAdd - } - return nil -} - -func (dc *dirtyCache) getMembersToDelete(setName string) map[string]struct{} { - diff, ok := dc.toCreateCache[setName] - if ok { - return diff.membersToDelete - } - diff, ok = dc.toUpdateCache[setName] - if ok { - return diff.membersToDelete - } - diff, ok = dc.toDestroyCache[setName] - if ok { - return diff.membersToDelete - } - return nil +func (diff *memberDiff) resetMembersToAdd() { + diff.membersToAdd = make(map[string]struct{}) } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux_test.go b/npm/pkg/dataplane/ipsets/dirtycache_linux_test.go new file mode 100644 index 0000000000..cee4ac3331 --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux_test.go @@ -0,0 +1,28 @@ +package ipsets + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func assertDiff(t *testing.T, expected testDiff, actual *memberDiff) { + if len(expected.toAdd) == 0 { + require.Equal(t, 0, len(actual.membersToAdd), "expected 0 members to add") + } else { + require.Equal(t, stringSliceToSet(expected.toAdd), actual.membersToAdd, "unexpected members to add for set") + } + if len(expected.toDelete) == 0 { + require.Equal(t, 0, len(actual.membersToDelete), "expected 0 members to delete") + } else { + require.Equal(t, stringSliceToSet(expected.toDelete), actual.membersToDelete, "unexpected members to delete for set") + } +} + +func stringSliceToSet(s []string) map[string]struct{} { + m := make(map[string]struct{}, len(s)) + for _, v := range s { + m[v] = struct{}{} + } + return m +} diff --git a/npm/pkg/dataplane/ipsets/dirtycache_test.go b/npm/pkg/dataplane/ipsets/dirtycache_test.go index af5999ac6f..744b2ba43c 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_test.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_test.go @@ -3,15 +3,13 @@ package ipsets import ( "testing" - "github.com/Azure/azure-container-networking/npm/util" "github.com/stretchr/testify/require" ) // members are only important for Linux type dirtyCacheResults struct { - // map of prefixed name to members to add/delete - toAddOrUpdate map[string]testDiff - // map of prefixed name to members to add/delete + toCreate map[string]testDiff + toUpdate map[string]testDiff toDestroy map[string]testDiff } @@ -21,7 +19,8 @@ type testDiff struct { } const ( - ip = "1.2.3.4" + ip1 = "1.2.3.4" + ip2 = "4.4.4.4" podKey = "pod1" ) @@ -32,8 +31,8 @@ func TestDirtyCacheReset(t *testing.T) { set4 := NewIPSet(NewIPSetMetadata("set4", Namespace)) dc := newDirtyCache() dc.create(set1) - dc.addMember(set2, ip) - dc.deleteMember(set3, "4.4.4.4") + dc.addMember(set2, ip1) + dc.deleteMember(set3, ip2) dc.destroy(set4) dc.reset() assertDirtyCache(t, dc, &dirtyCacheResults{}) @@ -46,15 +45,16 @@ func TestDirtyCacheResetAddOrUpdate(t *testing.T) { set4 := NewIPSet(NewIPSetMetadata("set4", Namespace)) dc := newDirtyCache() dc.create(set1) - dc.addMember(set2, ip) - dc.deleteMember(set3, "4.4.4.4") - set4.IPPodKey[ip] = podKey + dc.addMember(set2, ip1) + dc.deleteMember(set3, ip2) + // destroy will maintain this member to delete + set4.IPPodKey[ip1] = podKey dc.destroy(set4) dc.resetAddOrUpdateCache() assertDirtyCache(t, dc, &dirtyCacheResults{ toDestroy: map[string]testDiff{ set4.Name: { - toDelete: []string{ip}, + toDelete: []string{ip1}, }, }, }) @@ -65,165 +65,342 @@ func TestDirtyCacheCreate(t *testing.T) { dc := newDirtyCache() dc.create(set1) assertDirtyCache(t, dc, &dirtyCacheResults{ - toAddOrUpdate: map[string]testDiff{ + toCreate: map[string]testDiff{ set1.Name: {}, }, - toDestroy: nil, - }) -} - -// func TestDirtyCacheCreateAfterDelete(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// set1.IPPodKey[ip] = podKey -// dc.destroy(set1) -// // original members shouldn't get updated -// dc.create(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: map[string]*memberDiff{ -// set1.Name: {ip}, -// }, -// toDestroy: nil, -// }) -// } - -// func TestDirtyCacheUpdateNew(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: map[string][]string{ -// set1.Name: {ip}, -// }, -// toDestroy: nil, -// }) -// } - -// func TestDirtyCacheUpdateOld(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// dc.create(set1) -// // original members shouldn't get updated -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: map[string][]string{ -// set1.Name: {}, -// }, -// toDestroy: nil, -// }) -// } - -// func TestDirtyCacheUpdateTwice(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// dc.update(set1) -// // original members shouldn't get updated -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: map[string][]string{ -// set1.Name: {}, -// }, -// toDestroy: nil, -// }) -// } - -// func TestDirtyCacheUpdateAfterDelete(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// dc.destroy(set1) -// // original members shouldn't get updated -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: map[string][]string{ -// set1.Name: {}, -// }, -// toDestroy: nil, -// }) -// } - -// func TestDirtyCacheDelete(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// set1.IPPodKey[ip] = podKey -// dc.destroy(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: nil, -// toDestroy: map[string][]string{ -// set1.Name: {ip}, -// }, -// }) -// } - -// func TestDirtyCacheDeleteOld(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// // original members shouldn't get updated -// delete(set1.IPPodKey, ip) -// dc.destroy(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: nil, -// toDestroy: map[string][]string{ -// set1.Name: {ip}, -// }, -// }) -// } - -// func TestDirtyCacheDeleteTwice(t *testing.T) { -// set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) -// dc := newDirtyCache() -// set1.IPPodKey[ip] = podKey -// dc.update(set1) -// // original members shouldn't get updated -// delete(set1.IPPodKey, ip) -// dc.destroy(set1) -// assertDirtyCache(t, dc, &dirtyCacheResults{ -// toAddOrUpdate: nil, -// toDestroy: map[string][]string{ -// set1.Name: {ip}, -// }, -// }) -// } - -func assertDirtyCache(t *testing.T, dc dirtyCacheInterface, expected *dirtyCacheResults) { - require.Equal(t, len(expected.toAddOrUpdate), dc.numSetsToAddOrUpdate(), "unexpected number of sets to add or update") + }) +} + +func TestDirtyCacheCreateWithMembers(t *testing.T) { + // hash set + dc := newDirtyCache() + set1 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + set1.IPPodKey[ip1] = podKey + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, + }, + }) + + // list + dc.reset() + list := NewIPSet(NewIPSetMetadata("list", KeyValueLabelOfNamespace)) + list.MemberIPSets[set1.Name] = set1 + dc.create(list) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + list.Name: { + toAdd: []string{set1.HashedName}, + }, + }, + }) +} + +func TestDirtyCacheCreateIdempotence(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + // already created: this would create an error log + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + set1.Name: {}, + }, + }) + + dc.reset() + dc.addMember(set1, ip1) + // already updated: this would create an error log + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheCreateAfterDestroy(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + set1.IPPodKey[ip1] = podKey + dc.destroy(set1) + // maintain members to delete in Linux + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toDelete: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheDestroy(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.destroy(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDestroyWithMembers(t *testing.T) { + // hash set + dc := newDirtyCache() + set1 := NewIPSet(NewIPSetMetadata("set2", Namespace)) + set1.IPPodKey[ip1] = podKey + dc.destroy(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: { + toDelete: []string{ip1}, + }, + }, + }) + + // list + dc.reset() + list := NewIPSet(NewIPSetMetadata("list", KeyValueLabelOfNamespace)) + list.MemberIPSets[set1.Name] = set1 + dc.destroy(list) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + list.Name: { + toDelete: []string{set1.HashedName}, + }, + }, + }) +} + +func TestDirtyCacheDestroyIdempotence(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.destroy(set1) + // already destroyed: this would create an error log + dc.destroy(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDestroyAfterCreate(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + dc.addMember(set1, ip1) + dc.deleteMember(set1, ip2) + dc.destroy(set1) + // no set/diff to cache since the set was never in the kernel + assertDirtyCache(t, dc, &dirtyCacheResults{}) +} + +func TestDirtyCacheDestroyAfterAdd(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.addMember(set1, ip1) + dc.destroy(set1) + // assumes the set was in the kernel before + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDestroyAfterDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.deleteMember(set1, ip1) + dc.destroy(set1) + // assumes the set was in the kernel before + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: { + toDelete: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheAdd(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.addMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheAddIdempotence(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.addMember(set1, ip1) + // no error log + dc.addMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheAddAfterCreate(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + dc.addMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheAddAfterDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.deleteMember(set1, ip1) + dc.addMember(set1, ip1) + // in update cache despite no-diff + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheAddAfterDestroy(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.destroy(set1) + dc.addMember(set1, ip1) + // do nothing and create an error log + dc.deleteMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDelete(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.deleteMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toDelete: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheDeleteIdempotence(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.deleteMember(set1, ip1) + // no error log + dc.deleteMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: { + toDelete: []string{ip1}, + }, + }, + }) +} + +func TestDirtyCacheDeleteAfterCreate(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + dc.deleteMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDeleteAfterAdd(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.addMember(set1, ip1) + dc.deleteMember(set1, ip1) + // in update cache despite no-diff + assertDirtyCache(t, dc, &dirtyCacheResults{ + toUpdate: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheDeleteAfterDestroy(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.destroy(set1) + // do nothing and create an error log + dc.deleteMember(set1, ip1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toDestroy: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + +func TestDirtyCacheNumSetsToAddOrUpdate(t *testing.T) { + dc := newDirtyCache() + dc.toCreateCache["a"] = &memberDiff{} + dc.toCreateCache["b"] = &memberDiff{} + dc.toUpdateCache["c"] = &memberDiff{} + require.Equal(t, 3, dc.numSetsToAddOrUpdate()) +} + +func assertDirtyCache(t *testing.T, dc *dirtyCache, expected *dirtyCacheResults) { + require.Equal(t, len(expected.toCreate), len(dc.toCreateCache), "unexpected number of sets to create") + require.Equal(t, len(expected.toUpdate), len(dc.toUpdateCache), "unexpected number of sets to update") require.Equal(t, len(expected.toDestroy), dc.numSetsToDelete(), "unexpected number of sets to delete") - for setName, diff := range expected.toAddOrUpdate { + for setName, diff := range expected.toCreate { + actualDiff, ok := dc.toCreateCache[setName] + require.True(t, ok, "set %s not found in toCreateCache", setName) + require.NotNil(t, actualDiff, "member diff should not be nil for set %s", setName) + require.True(t, dc.isSetToAddOrUpdate(setName), "set %s should be added/updated", setName) + require.False(t, dc.isSetToDelete(setName), "set %s should not be deleted", setName) + // implemented in OS-specific test file + assertDiff(t, diff, dc.memberDiff(setName)) + } + for setName, diff := range expected.toUpdate { + actualDiff, ok := dc.toUpdateCache[setName] + require.True(t, ok, "set %s not found in toUpdateCache", setName) + require.NotNil(t, actualDiff, "member diff should not be nil for set %s", setName) require.True(t, dc.isSetToAddOrUpdate(setName), "set %s should be added/updated", setName) require.False(t, dc.isSetToDelete(setName), "set %s should not be deleted", setName) - assertDiff(t, dc, setName, diff) + // implemented in OS-specific test file + assertDiff(t, diff, dc.memberDiff(setName)) } for setName, diff := range expected.toDestroy { + require.NotNil(t, dc.toDestroyCache[setName], "member diff should not be nil for set %s", setName) require.True(t, dc.isSetToDelete(setName), "set %s should be deleted", setName) require.False(t, dc.isSetToAddOrUpdate(setName), "set %s should not be added/updated", setName) - assertDiff(t, dc, setName, diff) - } -} - -func assertDiff(t *testing.T, dc dirtyCacheInterface, setName string, diff testDiff) { - if !util.IsWindowsDP() { - if len(diff.toAdd) == 0 { - require.Equal(t, 0, len(dc.getMembersToAdd(setName)), "expected 0 members to add") - } else { - require.Equal(t, stringSliceToSet(diff.toAdd), dc.getMembersToAdd(setName), "unexpected members to add for set %s", setName) - } - if len(diff.toDelete) == 0 { - require.Equal(t, 0, len(dc.getMembersToDelete(setName)), "expected 0 members to delete") - } else { - require.Equal(t, stringSliceToSet(diff.toDelete), dc.getMembersToDelete(setName), "unexpected members to delete for set %s", setName) - } - } -} - -func stringSliceToSet(s []string) map[string]struct{} { - m := make(map[string]struct{}, len(s)) - for _, v := range s { - m[v] = struct{}{} + // implemented in OS-specific test file + assertDiff(t, diff, dc.memberDiff(setName)) } - return m } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index a0fbc4476f..290f721c97 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -2,95 +2,28 @@ package ipsets import "fmt" -type dirtyCache struct { - toAddOrUpdateCache map[string]struct{} - toDeleteCache map[string]struct{} -} - -func newDirtyCache() dirtyCacheInterface { - dc := &dirtyCache{} - dc.reset() - return dc -} - -func (dc *dirtyCache) reset() { - dc.toAddOrUpdateCache = make(map[string]struct{}) - dc.toDeleteCache = make(map[string]struct{}) -} - -func (dc *dirtyCache) resetAddOrUpdateCache() { - dc.toAddOrUpdateCache = make(map[string]struct{}) -} - -func (dc *dirtyCache) create(set *IPSet) { - putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) -} -func (dc *dirtyCache) addMember(set *IPSet, member string) { - putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) -} - -func (dc *dirtyCache) deleteMember(set *IPSet, member string) { - putInCacheAndRemoveFromOther(set, dc.toAddOrUpdateCache, dc.toDeleteCache) -} - -func (dc *dirtyCache) destroy(set *IPSet) { - putInCacheAndRemoveFromOther(set, dc.toDeleteCache, dc.toAddOrUpdateCache) -} - -func putInCacheAndRemoveFromOther(set *IPSet, intoCache, fromCache map[string]struct{}) { - if _, ok := intoCache[set.Name]; ok { - return - } - intoCache[set.Name] = struct{}{} - delete(fromCache, set.Name) -} - -func (dc *dirtyCache) getSetsToAddOrUpdate() map[string]struct{} { - m := make(map[string]struct{}, len(dc.toAddOrUpdateCache)) - for setName := range dc.toAddOrUpdateCache { - m[setName] = struct{}{} - } - return m -} - -func (dc *dirtyCache) getSetsToDelete() map[string]struct{} { - m := make(map[string]struct{}, len(dc.toDeleteCache)) - for setName := range dc.toDeleteCache { - m[setName] = struct{}{} - } - return m -} - -func (dc *dirtyCache) numSetsToAddOrUpdate() int { - return len(dc.toAddOrUpdateCache) -} - -func (dc *dirtyCache) numSetsToDelete() int { - return len(dc.toDeleteCache) -} +type *memberDiff struct{} -func (dc *dirtyCache) isSetToAddOrUpdate(setName string) bool { - _, ok := dc.toAddOrUpdateCache[setName] - return ok +func newMemberDiff() *memberDiff { + return &memberDiff{} } -func (dc *dirtyCache) isSetToDelete(setName string) bool { - _, ok := dc.toDeleteCache[setName] - return ok +func diffOnCreate(set *IPSet) *memberDiff { + return newMemberDiff() } -func (dc *dirtyCache) printAddOrUpdateCache() string { - return fmt.Sprintf("%+v", dc.toAddOrUpdateCache) +func (diff *memberDiff) addMember(member string) { + // no-op } -func (dc *dirtyCache) printDeleteCache() string { - return fmt.Sprintf("%+v", dc.toDeleteCache) +func (diff *memberDiff) deleteMember(member string) { + // no-op } -func (dc *dirtyCache) getMembersToAdd(setName string) map[string]struct{} { - return nil +func (diff *memberDiff) deleteMemberIfExists(member string) { + // no-op } -func (dc *dirtyCache) getMembersToDelete(setName string) map[string]struct{} { - return nil +func (diff *memberDiff) resetMembersToAdd() { + // no-op } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows_test.go b/npm/pkg/dataplane/ipsets/dirtycache_windows_test.go new file mode 100644 index 0000000000..313a0f6b8e --- /dev/null +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows_test.go @@ -0,0 +1,7 @@ +package ipsets + +import "testing" + +func assertDiff(_ *testing.T, _ testDiff, _ *memberDiff) { + // no-op +} diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index a44574f14e..10991be8e7 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -461,7 +461,7 @@ func (iMgr *IPSetManager) modifyCacheForCacheDeletion(set *IPSet, deleteOption u } delete(iMgr.setMap, set.Name) - metrics.DecNumIPSets() + metrics.DeleteIPSet(set.Name) if iMgr.iMgrCfg.IPSetMode == ApplyAllIPSets { // NOTE: in ApplyAllIPSets mode, if this ipset has never been created in the kernel, // it would be added to the deleteCache, and then the OS would fail to delete it @@ -527,7 +527,7 @@ func (iMgr *IPSetManager) modifyCacheForKernelMemberDelete(set *IPSet, member st // if so will not delete it func (iMgr *IPSetManager) sanitizeDirtyCache() { anyProblems := false - for setName := range iMgr.dirtyCache.getSetsToDelete() { + for setName := range iMgr.dirtyCache.setsToDelete() { if iMgr.dirtyCache.isSetToAddOrUpdate(setName) { klog.Errorf("[IPSetManager] Unexpected state in dirty cache %s set is part of both update and delete caches", setName) anyProblems = true diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 8f60a874bc..1592e83d70 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -321,7 +321,7 @@ func (iMgr *IPSetManager) fileCreatorForApplyWithSaveFile(maxTryCount int, saveF creator := ioutil.NewFileCreator(iMgr.ioShim, maxTryCount, ipsetRestoreLineFailurePattern) // TODO make the line failure pattern into a definition constant eventually // 1. create all sets first so we don't try to add a member set to a list if it hasn't been created yet - setsToAddOrUpdate := iMgr.dirtyCache.getSetsToAddOrUpdate() + setsToAddOrUpdate := iMgr.dirtyCache.setsToAddOrUpdate() for prefixedName := range setsToAddOrUpdate { set := iMgr.setMap[prefixedName] iMgr.createSetForApply(creator, set) @@ -355,7 +355,7 @@ func (iMgr *IPSetManager) fileCreatorForApplyWithSaveFile(maxTryCount int, saveF but until then, set A is in use by a kernel component and can't be destroyed. */ // flush all sets first in case a set we're destroying is referenced by a list we're destroying - setsToDelete := iMgr.dirtyCache.getSetsToDelete() + setsToDelete := iMgr.dirtyCache.setsToDelete() for prefixedName := range setsToDelete { iMgr.flushSetForApply(creator, prefixedName) } @@ -370,7 +370,7 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int) *ioutil.FileCreat creator := ioutil.NewFileCreator(iMgr.ioShim, maxTryCount, ipsetRestoreLineFailurePattern) // TODO make the line failure pattern into a definition constant eventually // 1. create all sets first so we don't try to add a member set to a list if it hasn't been created yet - setsToAddOrUpdate := iMgr.dirtyCache.getSetsToAddOrUpdate() + setsToAddOrUpdate := iMgr.dirtyCache.setsToAddOrUpdate() for prefixedName := range setsToAddOrUpdate { set := iMgr.setMap[prefixedName] iMgr.createSetForApply(creator, set) @@ -382,12 +382,11 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int) *ioutil.FileCreat for prefixedName := range setsToAddOrUpdate { sectionID := sectionID(addOrUpdateSectionPrefix, prefixedName) set := iMgr.setMap[prefixedName] - membersToDelete := iMgr.dirtyCache.getMembersToDelete(prefixedName) - for member := range membersToDelete { + diff := iMgr.dirtyCache.memberDiff(prefixedName) + for member := range diff.membersToDelete { iMgr.deleteMemberForApply(creator, set, sectionID, member) } - membersToAdd := iMgr.dirtyCache.getMembersToAdd(prefixedName) - for member := range membersToAdd { + for member := range diff.membersToAdd { iMgr.addMemberForApply(creator, set, sectionID, member) } } @@ -401,7 +400,7 @@ func (iMgr *IPSetManager) fileCreatorForApply(maxTryCount int) *ioutil.FileCreat but until then, set A is in use by a kernel component and can't be destroyed. */ // flush all sets first in case a set we're destroying is referenced by a list we're destroying - setsToDelete := iMgr.dirtyCache.getSetsToDelete() + setsToDelete := iMgr.dirtyCache.setsToDelete() for prefixedName := range setsToDelete { iMgr.flushSetForApply(creator, prefixedName) } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go index a8cff6fe97..0871bab74c 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux_test.go @@ -87,16 +87,6 @@ func TestApplyIPSets(t *testing.T) { expectedExecCount: 1, wantErr: false, }, - { - name: "set is in both delete and add/update cache", - args: args{ - toAddUpdateSets: []*IPSetMetadata{namespaceSet}, - toDeleteSets: []*IPSetMetadata{namespaceSet}, - commandError: false, - }, - expectedExecCount: 1, - wantErr: false, - }, { name: "apply error", args: args{ @@ -458,17 +448,27 @@ func TestResetIPSetsOnFailure(t *testing.T) { }) } +// for applyIPSetsWithSave() func TestApplyIPSetsSuccessWithoutSave(t *testing.T) { - // no sets to add/update, so don't call ipset save - calls := []testutils.TestCmd{{Cmd: ipsetRestoreStringSlice}} + calls := []testutils.TestCmd{ + {Cmd: ipsetSaveStringSlice, PipedToCommand: true}, + {Cmd: []string{"grep", "azure-npm-"}}, + {Cmd: ipsetRestoreStringSlice}, + {Cmd: ipsetRestoreStringSlice}, + } ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) // delete a set so the file isn't empty (otherwise the creator won't even call the exec command) iMgr.CreateIPSets([]*IPSetMetadata{TestNSSet.Metadata}) // create so we can delete + err := iMgr.applyIPSetsWithSaveFile() + require.NoError(t, err) + iMgr.clearDirtyCache() + + // no sets to add/update, so don't call ipset save iMgr.DeleteIPSet(TestNSSet.PrefixName, util.SoftDelete) - err := iMgr.applyIPSets() + err = iMgr.applyIPSetsWithSaveFile() require.NoError(t, err) } @@ -665,12 +665,18 @@ func TestDestroy(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - // without save file - calls := []testutils.TestCmd{fakeRestoreSuccessCommand} + calls := []testutils.TestCmd{ + fakeRestoreSuccessCommand, + } ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + // remove some members and destroy some sets require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) @@ -678,16 +684,14 @@ func TestDestroy(t *testing.T) { iMgr.CreateIPSets([]*IPSetMetadata{TestKeyPodSet.Metadata}) require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) require.NoError(t, iMgr.RemoveFromList(TestKeyNSList.Metadata, []*IPSetMetadata{TestKeyPodSet.Metadata})) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) var creator *ioutil.FileCreator if tt.withSaveFile { - creator = iMgr.fileCreatorForApplyWithSaveFile(len(calls), nil) + creator = iMgr.fileCreatorForApplyWithSaveFile(1, nil) } else { - creator = iMgr.fileCreatorForApply(len(calls)) + creator = iMgr.fileCreatorForApply(1) } actualLines := testAndSortRestoreFileString(t, creator.ToString()) @@ -713,6 +717,52 @@ func TestDestroy(t *testing.T) { } } +// no save file involved +func TestDeleteMembers(t *testing.T) { + calls := []testutils.TestCmd{ + fakeRestoreSuccessCommand, + } + ioshim := common.NewMockIOShim(calls) + defer ioshim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "1.1.1.1", "a")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "2.2.2.2", "b")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "3.3.3.3", "c")) + // create to destroy later + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + + // will remove this member + require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "1.1.1.1", "a")) + // will add this member + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "5.5.5.5", "e")) + // won't add/remove this member since the next two calls cancel each other out + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "4.4.4.4", "d")) + require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "4.4.4.4", "d")) + // won't add/remove this member since the next two calls cancel each other out + require.NoError(t, iMgr.RemoveFromSets([]*IPSetMetadata{TestNSSet.Metadata}, "2.2.2.2", "b")) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "2.2.2.2", "b")) + // destroy extra set + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) + + expectedLines := []string{ + fmt.Sprintf("-N %s --exist nethash", TestNSSet.HashedName), + fmt.Sprintf("-D %s 1.1.1.1", TestNSSet.HashedName), + fmt.Sprintf("-A %s 5.5.5.5", TestNSSet.HashedName), + fmt.Sprintf("-F %s", TestCIDRSet.HashedName), + fmt.Sprintf("-X %s", TestCIDRSet.HashedName), + "", + } + sortedExpectedLines := testAndSortRestoreFileLines(t, expectedLines) + creator := iMgr.fileCreatorForApply(len(calls)) + actualLines := testAndSortRestoreFileString(t, creator.ToString()) + dptestutils.AssertEqualLines(t, sortedExpectedLines, actualLines) + wasFileAltered, err := creator.RunCommandOnceWithFile("ipset", "restore") + require.NoError(t, err, "ipset restore should be successful") + require.False(t, wasFileAltered, "file should not be altered") +} + func TestUpdateWithIdenticalSaveFile(t *testing.T) { calls := []testutils.TestCmd{fakeRestoreSuccessCommand} ioshim := common.NewMockIOShim(calls) @@ -793,6 +843,10 @@ func TestUpdateWithRealisticSaveFile(t *testing.T) { saveFileString := strings.Join(saveFileLines, "\n") saveFileBytes := []byte(saveFileString) + iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.1", "b")) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.2", "c")) @@ -803,7 +857,6 @@ func TestUpdateWithRealisticSaveFile(t *testing.T) { require.NoError(t, iMgr.AddToLists([]*IPSetMetadata{TestKeyNSList.Metadata}, []*IPSetMetadata{TestNSSet.Metadata, TestKeyPodSet.Metadata})) iMgr.CreateIPSets([]*IPSetMetadata{TestKVNSList.Metadata}) require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestCIDRSet.Metadata}, "1.2.3.4", "z")) // set not in save file - iMgr.CreateIPSets([]*IPSetMetadata{TestNestedLabelList.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestNestedLabelList.PrefixName, util.SoftDelete) creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) @@ -1341,11 +1394,14 @@ func TestFailureOnFlush(t *testing.T) { saveFileString := strings.Join(saveFileLines, "\n") saveFileBytes := []byte(saveFileString) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) @@ -1394,11 +1450,14 @@ func TestFailureOnDestroy(t *testing.T) { saveFileString := strings.Join(saveFileLines, "\n") saveFileBytes := []byte(saveFileString) + iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestNSSet.Metadata}, "10.0.0.0", "a")) // in kernel already require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{TestKeyPodSet.Metadata}, "10.0.0.0", "a")) // not in kernel yet - iMgr.CreateIPSets([]*IPSetMetadata{TestKVPodSet.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestKVPodSet.PrefixName, util.SoftDelete) - iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) creator := iMgr.fileCreatorForApplyWithSaveFile(len(calls), saveFileBytes) @@ -1446,6 +1505,9 @@ func TestFailureOnLastLine(t *testing.T) { iMgr := NewIPSetManager(applyAlwaysCfg, ioshim) iMgr.CreateIPSets([]*IPSetMetadata{TestCIDRSet.Metadata}) // create so we can delete + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + iMgr.DeleteIPSet(TestCIDRSet.PrefixName, util.SoftDelete) var creator *ioutil.FileCreator diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go index f5e349e3a3..51d35a86fc 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_test.go @@ -404,6 +404,53 @@ func TestDeleteIPSet(t *testing.T) { } } +func TestDeleteAfterCreate(t *testing.T) { + metrics.ReinitializeAll() + calls := []testutils.TestCmd{} + ioShim := common.NewMockIOShim(calls) + defer ioShim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyOnNeedCfg, ioShim) + + setMetadata := NewIPSetMetadata(testSetName, Namespace) + iMgr.CreateIPSets([]*IPSetMetadata{setMetadata}) + iMgr.DeleteIPSet(setMetadata.GetPrefixName(), util.SoftDelete) + assertExpectedInfo(t, iMgr, &expectedInfo{}) +} + +func TestCreateAfterHardDelete(t *testing.T) { + metrics.ReinitializeAll() + calls := []testutils.TestCmd{} + ioShim := common.NewMockIOShim(calls) + defer ioShim.VerifyCalls(t, calls) + iMgr := NewIPSetManager(applyAlwaysCfg, ioShim) + + setMetadata := NewIPSetMetadata(testSetName, Namespace) + require.NoError(t, iMgr.AddToSets([]*IPSetMetadata{setMetadata}, "1.2.3.4", "pod-a")) + // clear dirty cache, otherwise a set deletion will be a no-op + iMgr.clearDirtyCache() + + numIPSetsInCache, _ := metrics.GetNumIPSets() + fmt.Println(numIPSetsInCache) + + iMgr.DeleteIPSet(setMetadata.GetPrefixName(), util.ForceDelete) + numIPSetsInCache, _ = metrics.GetNumIPSets() + fmt.Println(numIPSetsInCache) + + assertExpectedInfo(t, iMgr, &expectedInfo{ + toDeleteCache: []string{setMetadata.GetPrefixName()}, + }) + + iMgr.CreateIPSets([]*IPSetMetadata{setMetadata}) + assertExpectedInfo(t, iMgr, &expectedInfo{ + mainCache: []setMembers{ + { + metadata: setMetadata, + }, + }, + toAddUpdateCache: []*IPSetMetadata{setMetadata}, + }) +} + func TestDeleteIPSetNotAllowed(t *testing.T) { // try to delete a list with a member and a set referenced in kernel (by a list) // must use applyAlwaysCfg for set to be in kernel @@ -1427,7 +1474,7 @@ func assertExpectedInfo(t *testing.T, iMgr *IPSetManager, info *expectedInfo) { // TODO update get function when we have prometheus metric for in kernel numIPSetsInCache, err := metrics.GetNumIPSets() promutil.NotifyIfErrors(t, err) - require.Equal(t, len(iMgr.setMap), numIPSetsInCache, "numIPSetsInCache mismatch") + require.Equal(t, len(iMgr.setMap), numIPSetsInCache, "num ipsets mismatch") // the setMap is equal expectedNumEntriesInCache := 0 @@ -1449,7 +1496,7 @@ func assertExpectedInfo(t *testing.T, iMgr *IPSetManager, info *expectedInfo) { // TODO update get func when we have prometheus metric for in kernel numEntriesInCache, err := metrics.GetNumIPSetEntries() promutil.NotifyIfErrors(t, err) - require.Equal(t, expectedNumEntriesInCache, numEntriesInCache) + require.Equal(t, expectedNumEntriesInCache, numEntriesInCache, "incorrect num ipset entries") for _, set := range iMgr.setMap { expectedNumEntries := 0 // TODO replace bool with iMgr.shouldBeInKernel(set) when we have prometheus metric for in kernel diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index cfd3b5e74e..a1a5889dc5 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -157,7 +157,7 @@ func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkP } existingSets, toDeleteSets := iMgr.segregateSetPolicies(networkPolicies, donotResetIPSets) // some of this below logic can be abstracted a step above - toAddUpdateSetNames := iMgr.dirtyCache.getSetsToAddOrUpdate() + toAddUpdateSetNames := iMgr.dirtyCache.setsToAddOrUpdate() setPolicyBuilder.toDeleteSets = toDeleteSets // for faster look up changing a slice to map From f7e10970328bee020ee006a357161a8a496acd90 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Wed, 13 Apr 2022 15:25:15 -0700 Subject: [PATCH 15/17] fix windows build --- npm/pkg/dataplane/ipsets/dirtycache_windows.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 290f721c97..25bfa827af 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -1,8 +1,6 @@ package ipsets -import "fmt" - -type *memberDiff struct{} +type memberDiff struct{} func newMemberDiff() *memberDiff { return &memberDiff{} From 7b93dc52829911286881501f2d6f912db44932fe Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Thu, 14 Apr 2022 11:18:35 -0700 Subject: [PATCH 16/17] change some error situations to valid situations --- npm/pkg/dataplane/ipsets/dirtycache.go | 64 +++++++++------------ npm/pkg/dataplane/ipsets/dirtycache_test.go | 58 +++++++++++++------ 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index d0f125c7c3..93e49b6efe 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -16,15 +16,19 @@ import ( Assumptions: - if the set becomes dirty via update or destroy, then the set WAS in the kernel before - - update won't be called unless the set IS in the kernel OR create was called before - - destroy won't be called unless the set IS in the kernel OR create was called before - if the set becomes dirty via create, then the set was NOT in the kernel before - - create won't be called twice (unless a destroy call comes in between) + + Usage: + - create, addMember, deleteMember, and destroy are idempotent + - create should not be called if the set becomes dirty via add/delete or the set is removed from the deleteCache via add/update + - deleteMember should not be called if the set is in the deleteCache + - deleteMember is safe to call on members in the kernel and members added via addMember + - deleteMember is also safe to call on members not in the kernel if the set isn't in the kernel yet (became dirty via create) Examples of Expected Behavior: - if a set is created and then destroyed, that set will not be in the dirty cache anymore - if a set is updated and then destroyed, that set will be in the delete cache - - if a the only operations on a set are adding and removing the same member, the set may still be in the dirty cache, but the member will be untracked + - if the only operations on a set are adding and removing the same member, the set may still be in the dirty cache, but the member will be untracked */ type dirtyCacheInterface interface { // reset empties dirty cache @@ -85,16 +89,13 @@ func (dc *dirtyCache) resetAddOrUpdateCache() { } func (dc *dirtyCache) create(set *IPSet) { - // error checking if _, ok := dc.toCreateCache[set.Name]; ok { - msg := fmt.Sprintf("create: set %s should not already be in the toCreateCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) return } + // error checking if _, ok := dc.toUpdateCache[set.Name]; ok { - msg := fmt.Sprintf("create: set %s should not be in the toUpdateCache", set.Name) - klog.Error(msg) + msg := fmt.Sprintf("create should not be called for set %s since it's in the toUpdateCache", set.Name) + klog.Warning(msg) metrics.SendErrorLogAndMetric(util.IpsmID, msg) return } @@ -113,56 +114,46 @@ func (dc *dirtyCache) create(set *IPSet) { // could optimize Linux to remove from toUpdateCache if there were no member diff afterwards, // but leaving as is prevents difference between OS caches func (dc *dirtyCache) addMember(set *IPSet, member string) { - // error checking - if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("addMember: set %s should not be in the toDestroyCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) - return - } - diff, ok := dc.toCreateCache[set.Name] if !ok { diff, ok = dc.toUpdateCache[set.Name] if !ok { - diff = newMemberDiff() - dc.toUpdateCache[set.Name] = diff + diff, ok = dc.toDestroyCache[set.Name] + if !ok { + diff = newMemberDiff() + } } + dc.toUpdateCache[set.Name] = diff } + delete(dc.toDestroyCache, set.Name) diff.addMember(member) } // could optimize Linux to remove from toUpdateCache if there were no member diff afterwards, // but leaving as is prevents difference between OS caches func (dc *dirtyCache) deleteMember(set *IPSet, member string) { - // error checking + // error checking #1 if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("deleteMember: set %s should not be in the toDestroyCache", set.Name) - klog.Error(msg) + msg := fmt.Sprintf("attempting to delete member %s for set %s in the toDestroyCache", member, set.Name) + klog.Warning(msg) metrics.SendErrorLogAndMetric(util.IpsmID, msg) return } - - diff, ok := dc.toCreateCache[set.Name] - if ok { + if diff, ok := dc.toCreateCache[set.Name]; ok { // don't mark a member to be deleted if it never existed in the kernel diff.deleteMemberIfExists(member) } else { - diff, ok = dc.toUpdateCache[set.Name] + diff, ok := dc.toUpdateCache[set.Name] if !ok { diff = newMemberDiff() - dc.toUpdateCache[set.Name] = diff } + dc.toUpdateCache[set.Name] = diff diff.deleteMember(member) } } func (dc *dirtyCache) destroy(set *IPSet) { - // error checking if dc.isSetToDelete(set.Name) { - msg := fmt.Sprintf("destroy: set %s should not already be in the toDestroyCache", set.Name) - klog.Error(msg) - metrics.SendErrorLogAndMetric(util.IpsmID, msg) return } @@ -248,16 +239,13 @@ func (dc *dirtyCache) printDeleteCache() string { } func (dc *dirtyCache) memberDiff(setName string) *memberDiff { - diff, ok := dc.toCreateCache[setName] - if ok { + if diff, ok := dc.toCreateCache[setName]; ok { return diff } - diff, ok = dc.toUpdateCache[setName] - if ok { + if diff, ok := dc.toUpdateCache[setName]; ok { return diff } - diff, ok = dc.toDestroyCache[setName] - if ok { + if diff, ok := dc.toDestroyCache[setName]; ok { return diff } return newMemberDiff() diff --git a/npm/pkg/dataplane/ipsets/dirtycache_test.go b/npm/pkg/dataplane/ipsets/dirtycache_test.go index 744b2ba43c..48b19e3452 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_test.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_test.go @@ -19,8 +19,8 @@ type testDiff struct { } const ( - ip1 = "1.2.3.4" - ip2 = "4.4.4.4" + ip1 = "1.1.1.1" + ip2 = "2.2.2.2" podKey = "pod1" ) @@ -99,31 +99,46 @@ func TestDirtyCacheCreateWithMembers(t *testing.T) { }) } -func TestDirtyCacheCreateIdempotence(t *testing.T) { +func TestDirtyCacheCreateAfterAddOrDelete(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() - dc.create(set1) - // already created: this would create an error log + dc.addMember(set1, ip1) + // already updated: this would create a warning log dc.create(set1) assertDirtyCache(t, dc, &dirtyCacheResults{ - toCreate: map[string]testDiff{ - set1.Name: {}, + toUpdate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip1}, + }, }, }) dc.reset() - dc.addMember(set1, ip1) - // already updated: this would create an error log + dc.deleteMember(set1, ip1) + // already updated: this would create a warning log dc.create(set1) assertDirtyCache(t, dc, &dirtyCacheResults{ toUpdate: map[string]testDiff{ set1.Name: { - toAdd: []string{ip1}, + toDelete: []string{ip1}, }, }, }) } +func TestDirtyCacheCreateIdempotence(t *testing.T) { + set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) + dc := newDirtyCache() + dc.create(set1) + // already created: no warning log + dc.create(set1) + assertDirtyCache(t, dc, &dirtyCacheResults{ + toCreate: map[string]testDiff{ + set1.Name: {}, + }, + }) +} + func TestDirtyCacheCreateAfterDestroy(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() @@ -248,7 +263,7 @@ func TestDirtyCacheAddIdempotence(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() dc.addMember(set1, ip1) - // no error log + // no warning log dc.addMember(set1, ip1) assertDirtyCache(t, dc, &dirtyCacheResults{ toUpdate: map[string]testDiff{ @@ -289,13 +304,15 @@ func TestDirtyCacheAddAfterDelete(t *testing.T) { func TestDirtyCacheAddAfterDestroy(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() - dc.destroy(set1) - dc.addMember(set1, ip1) - // do nothing and create an error log dc.deleteMember(set1, ip1) + dc.destroy(set1) + dc.addMember(set1, ip2) assertDirtyCache(t, dc, &dirtyCacheResults{ - toDestroy: map[string]testDiff{ - set1.Name: {}, + toUpdate: map[string]testDiff{ + set1.Name: { + toAdd: []string{ip2}, + toDelete: []string{ip1}, + }, }, }) } @@ -356,12 +373,15 @@ func TestDirtyCacheDeleteAfterAdd(t *testing.T) { func TestDirtyCacheDeleteAfterDestroy(t *testing.T) { set1 := NewIPSet(NewIPSetMetadata("set1", Namespace)) dc := newDirtyCache() - dc.destroy(set1) - // do nothing and create an error log dc.deleteMember(set1, ip1) + dc.destroy(set1) + // do nothing and create a warning log + dc.deleteMember(set1, ip2) assertDirtyCache(t, dc, &dirtyCacheResults{ toDestroy: map[string]testDiff{ - set1.Name: {}, + set1.Name: { + toDelete: []string{ip1}, + }, }, }) } From 72e6bc85d20e0d51de821afc3fd0a55736d72ec0 Mon Sep 17 00:00:00 2001 From: Hunter Gregory Date: Thu, 14 Apr 2022 17:50:18 -0700 Subject: [PATCH 17/17] log clean up and addressing comments --- npm/pkg/dataplane/ipsets/dirtycache.go | 4 ++-- npm/pkg/dataplane/ipsets/dirtycache_linux.go | 2 +- npm/pkg/dataplane/ipsets/dirtycache_windows.go | 2 +- npm/pkg/dataplane/ipsets/ipsetmanager.go | 4 ++-- npm/pkg/dataplane/ipsets/ipsetmanager_linux.go | 1 + 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/npm/pkg/dataplane/ipsets/dirtycache.go b/npm/pkg/dataplane/ipsets/dirtycache.go index 93e49b6efe..ae35486ef2 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache.go +++ b/npm/pkg/dataplane/ipsets/dirtycache.go @@ -141,7 +141,7 @@ func (dc *dirtyCache) deleteMember(set *IPSet, member string) { } if diff, ok := dc.toCreateCache[set.Name]; ok { // don't mark a member to be deleted if it never existed in the kernel - diff.deleteMemberIfExists(member) + diff.removeMemberFromDiffToAdd(member) } else { diff, ok := dc.toUpdateCache[set.Name] if !ok { @@ -231,7 +231,7 @@ func (dc *dirtyCache) printAddOrUpdateCache() string { for setName, diff := range dc.toUpdateCache { toUpdate = append(toUpdate, fmt.Sprintf("%s: %+v", setName, diff)) } - return fmt.Sprintf("[to create: %+v], [to update: %+v]", strings.Join(toCreate, ","), strings.Join(toUpdate, ",")) + return fmt.Sprintf("to create: [%+v], to update: [%+v]", strings.Join(toCreate, ","), strings.Join(toUpdate, ",")) } func (dc *dirtyCache) printDeleteCache() string { diff --git a/npm/pkg/dataplane/ipsets/dirtycache_linux.go b/npm/pkg/dataplane/ipsets/dirtycache_linux.go index e3ba14f90c..e8433b1b02 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_linux.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_linux.go @@ -50,7 +50,7 @@ func (diff *memberDiff) deleteMember(member string) { } } -func (diff *memberDiff) deleteMemberIfExists(member string) { +func (diff *memberDiff) removeMemberFromDiffToAdd(member string) { delete(diff.membersToAdd, member) } diff --git a/npm/pkg/dataplane/ipsets/dirtycache_windows.go b/npm/pkg/dataplane/ipsets/dirtycache_windows.go index 25bfa827af..54066074bf 100644 --- a/npm/pkg/dataplane/ipsets/dirtycache_windows.go +++ b/npm/pkg/dataplane/ipsets/dirtycache_windows.go @@ -18,7 +18,7 @@ func (diff *memberDiff) deleteMember(member string) { // no-op } -func (diff *memberDiff) deleteMemberIfExists(member string) { +func (diff *memberDiff) removeMemberFromDiffToAdd(member string) { // no-op } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index 10991be8e7..800f2fe3fc 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -74,7 +74,7 @@ func (iMgr *IPSetManager) Reconcile() { } numRemovedSets := originalNumSets - len(iMgr.setMap) if numRemovedSets > 0 { - klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache (in Linux, includes original members) to: %+v", numRemovedSets, iMgr.dirtyCache.printDeleteCache()) + klog.Infof("[IPSetManager] removed %d empty/unreferenced ipsets, updating toDeleteCache to: %+v", numRemovedSets, iMgr.dirtyCache.printDeleteCache()) } } @@ -412,7 +412,7 @@ func (iMgr *IPSetManager) ApplyIPSets() error { } klog.Infof( - "[IPSetManager] dirty caches (in Linux, includes original members). toAddUpdateCache: %s \ntoDeleteCache: %s", + "[IPSetManager] dirty caches. toAddUpdateCache: %s, toDeleteCache: %s", iMgr.dirtyCache.printAddOrUpdateCache(), iMgr.dirtyCache.printDeleteCache(), ) iMgr.sanitizeDirtyCache() diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 1592e83d70..5d732ac59a 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -245,6 +245,7 @@ example where every set in add/update cache should have ip 1.2.3.4 and 2.3.4.5: -X set-to-delete3 -X set-to-delete1 */ +// unused currently, but may be used later for reconciling with the kernel func (iMgr *IPSetManager) applyIPSetsWithSaveFile() error { var saveFile []byte var saveError error