fix: reduce memory usage during MergeFiles

entryDetail.go

@@ -634,25 +634,3 @@ func sortEntriesByTraceNumber(entries []*EntryDetail) []*EntryDetail {
 	})
 	return entries
 }
-
-type traceNumbers []string
-
-func (nums traceNumbers) contains(num string) bool {
-	for i := range nums {
-		if num == nums[i] {
-			return true
-		}
-	}
-	return false
-}
-
-func getTraceNumbers(f *File) traceNumbers {
-	var out []string
-	for i := range f.Batches {
-		entries := f.Batches[i].GetEntries()
-		for j := range entries {
-			out = append(out, entries[j].TraceNumber)
-		}
-	}
-	return traceNumbers(out)
-}

entryDetail_test.go

@@ -28,8 +28,6 @@ import (
 	"testing"
 
 	"github.com/moov-io/base"
-
-	"github.com/stretchr/testify/require"
 )
 
 // mockEntryDetail creates an entry detail
@@ -774,20 +772,3 @@ func TestEntryDetail__LargeAmountStrings(t *testing.T) {
 		}
 	}
 }
-
-func TestGetTraceNumbers(t *testing.T) {
-	file := mockFilePPD()
-	odfi := mockBatchHeader().ODFIIdentification
-
-	trials := 10000
-
-	for i := 1; i < trials; i++ {
-		ed := mockEntryDetail()
-		ed.SetTraceNumber(odfi, i)
-
-		file.Batches[0].AddEntry(ed)
-	}
-
-	numbers := getTraceNumbers(file)
-	require.Equal(t, trials, len(numbers))
-}

go.sum

@@ -1552,6 +1552,7 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
 golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

merge.go

@@ -163,8 +163,6 @@ next:
 //
 // findOutfile will return the existing file (stored in outfiles) if no matching file exists.
 func (fs *mergableFiles) findOutfile(f *File) *File {
-	inTraceNumbers := getTraceNumbers(f)
-
 	var lookup func(int) *File
 	lookup = func(start int) *File {
 		// To allow recursive lookups we need to memorize the current index so deeper calls
@@ -174,13 +172,21 @@ func (fs *mergableFiles) findOutfile(f *File) *File {
 				fs.outfiles[i].Header.ImmediateOrigin == f.Header.ImmediateOrigin {
 
 				// found a matching file, so verify the TraceNumber isn't alreay inside
-				outTraceNumbers := getTraceNumbers(fs.outfiles[i])
-				for _, trace := range inTraceNumbers {
-					// If any of our incoming trace numbers match the existing merged file
-					// return the entire file as separate. This keeps partially overlapping
-					// batches self-contained.
-					if outTraceNumbers.contains(trace) {
-						return lookup(i + 1)
+				for inB := range f.Batches {
+					inEntries := f.Batches[inB].GetEntries()
+					for inE := range inEntries {
+						// Compare against outfiles
+						for outB := range fs.outfiles[i].Batches {
+							outEntries := fs.outfiles[i].Batches[outB].GetEntries()
+							for outE := range outEntries {
+								// If any of our incoming trace numbers match the existing merged file
+								// return the entire file as separate. This keeps partially overlapping
+								// batches self-contained.
+								if inEntries[inE].TraceNumber == outEntries[outE].TraceNumber {
+									return lookup(i + 1)
+								}
+							}
+						}
 					}
 				}
 

merge_test.go

@@ -400,8 +400,10 @@ func TestMergeFiles__dollarAmount2(t *testing.T) {
 
 func countTraceNumbers(files ...*File) int {
 	var total int
-	for i := range files {
-		total += len(getTraceNumbers(files[i]))
+	for f := range files {
+		for b := range files[f].Batches {
+			total += len(files[f].Batches[b].GetEntries())
+		}
 	}
 	return total
 }

test/issues/issue927_test.go

@@ -35,7 +35,7 @@ func TestIssue927(t *testing.T) {
 	}
 
 	if len(after) != 2 {
-		t.Errorf("merged %d files into %d", len(before), len(after))
+		t.Fatalf("merged %d files into %d files", len(before), len(after))
 	}
 
 	if n := len(after[0].Batches); n != 5 {