From 591c64f7fb3eab9022462daab97d17fdd0fef0a3 Mon Sep 17 00:00:00 2001 From: Jian Guan <61915096+tdq45gj@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:12:06 -0400 Subject: [PATCH 1/4] Update migration_verifier.go --- main/migration_verifier.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/main/migration_verifier.go b/main/migration_verifier.go index 8513b2d5..19faa3d9 100644 --- a/main/migration_verifier.go +++ b/main/migration_verifier.go @@ -6,6 +6,7 @@ import ( "math" _ "net/http/pprof" "os" + "runtime/pprof" "strings" "time" @@ -48,6 +49,8 @@ func main() { zerolog.SetGlobalLevel(zerolog.InfoLevel) ctx := context.TODO() + startPeriodicHeapProfileCollection(ctx) + flags := []cli.Flag{ altsrc.NewStringFlag(cli.StringFlag{ Name: configFileFlag, @@ -253,3 +256,35 @@ func expandCommaSeparators(in []string) []string { } return ret } + +func startPeriodicHeapProfileCollection(ctx context.Context) { + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + collectHeapUsage() + } + } + }() + +} + +func collectHeapUsage() { + heapFileName := fmt.Sprintf("heap-%s.out", time.Now().UTC().Format("20060102T150405Z")) + heapFile, err := os.Create(heapFileName) + defer heapFile.Close() + + if err != nil { + panic(err) + } + + err = pprof.Lookup("heap").WriteTo(heapFile, 0) + if err != nil { + heapFile.WriteString(err.Error()) + } +} From b7e980522b4b2100acf1f52f00334d653af906c9 Mon Sep 17 00:00:00 2001 From: Jian Guan <61915096+tdq45gj@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:13:20 -0400 Subject: [PATCH 2/4] Update migration_verifier.go --- main/migration_verifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/migration_verifier.go b/main/migration_verifier.go index 19faa3d9..77937eb9 100644 --- a/main/migration_verifier.go +++ b/main/migration_verifier.go @@ -259,7 +259,7 @@ func expandCommaSeparators(in []string) []string { func startPeriodicHeapProfileCollection(ctx context.Context) { go func() { - ticker := time.NewTicker(30 * time.Second) + ticker := time.NewTicker(10 * time.Minute) defer ticker.Stop() for { From 95bc9eadd47ed162e6eaadfd222f5b2f0dd74adf Mon Sep 17 00:00:00 2001 From: Jian Guan <61915096+tdq45gj@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:29:28 -0400 Subject: [PATCH 3/4] add flag to control collect interval --- internal/verifier/check.go | 1 + internal/verifier/migration_verifier.go | 16 +++++++++ internal/verifier/pprof.go | 45 +++++++++++++++++++++++++ main/migration_verifier.go | 41 ++++------------------ 4 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 internal/verifier/pprof.go diff --git a/internal/verifier/check.go b/internal/verifier/check.go index be64d92a..b7a1338d 100644 --- a/internal/verifier/check.go +++ b/internal/verifier/check.go @@ -40,6 +40,7 @@ func (verifier *Verifier) Check(ctx context.Context, filter map[string]any) { verifier.logger.Fatal().Err(err).Msgf("Fatal error in generation %d", verifier.generation) } }() + verifier.MaybeStartPeriodicHeapProfileCollection(ctx) } func (verifier *Verifier) waitForChangeStream() error { diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go index 711ed9f4..9271634b 100644 --- a/internal/verifier/migration_verifier.go +++ b/internal/verifier/migration_verifier.go @@ -136,6 +136,8 @@ type Verifier struct { // The filter is applied to all namespaces in both initial checking and iterative checking. // The verifier only checks documents within the filter. globalFilter map[string]any + + pprofInterval time.Duration } // VerificationStatus holds the Verification Status @@ -360,6 +362,20 @@ func (verifier *Verifier) SetReadPreference(arg string) error { return err } +func (verifier *Verifier) SetPprofInterval(arg string) error { + if arg == "" { + return nil + } + + interval, err := time.ParseDuration(arg) + if err != nil { + return err + } + + verifier.pprofInterval = interval + return nil +} + // DocumentStats gets various stats (TODO clarify) func DocumentStats(ctx context.Context, client *mongo.Client, namespaces []string) { diff --git a/internal/verifier/pprof.go b/internal/verifier/pprof.go new file mode 100644 index 00000000..f0876392 --- /dev/null +++ b/internal/verifier/pprof.go @@ -0,0 +1,45 @@ +package verifier + +import ( + "context" + "fmt" + "os" + "runtime/pprof" + "time" +) + +func (verifier *Verifier) MaybeStartPeriodicHeapProfileCollection(ctx context.Context) { + if verifier.pprofInterval == 0 { + return + } + + go func() { + ticker := time.NewTicker(verifier.pprofInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + collectHeapUsage() + } + } + }() + +} + +func collectHeapUsage() { + heapFileName := fmt.Sprintf("heap-%s.out", time.Now().UTC().Format("20060102T150405Z")) + heapFile, err := os.Create(heapFileName) + defer heapFile.Close() + + if err != nil { + panic(err) + } + + err = pprof.Lookup("heap").WriteTo(heapFile, 0) + if err != nil { + heapFile.WriteString(err.Error()) + } +} diff --git a/main/migration_verifier.go b/main/migration_verifier.go index 77937eb9..787218fd 100644 --- a/main/migration_verifier.go +++ b/main/migration_verifier.go @@ -6,7 +6,6 @@ import ( "math" _ "net/http/pprof" "os" - "runtime/pprof" "strings" "time" @@ -41,6 +40,7 @@ const ( failureDisplaySize = "failureDisplaySize" ignoreReadConcernFlag = "ignoreReadConcern" configFileFlag = "configFile" + pprofInterval = "pprofInterval" ) func main() { @@ -49,8 +49,6 @@ func main() { zerolog.SetGlobalLevel(zerolog.InfoLevel) ctx := context.TODO() - startPeriodicHeapProfileCollection(ctx) - flags := []cli.Flag{ altsrc.NewStringFlag(cli.StringFlag{ Name: configFileFlag, @@ -149,6 +147,10 @@ func main() { Name: ignoreReadConcernFlag, Usage: "Use connection-default read concerns rather than setting majority read concern. This option may degrade consistency, so only enable it if majority read concern (the default) doesn’t work.", }), + altsrc.NewStringFlag(cli.StringFlag{ + Name: pprofInterval, + Usage: "Interval to periodically collect pprof profiles", + }), } app := &cli.App{ @@ -210,6 +212,7 @@ func handleArgs(ctx context.Context, cCtx *cli.Context) (*verifier.Verifier, err v.SetNumWorkers(cCtx.Int(numWorkers)) v.SetGenerationPauseDelayMillis(time.Duration(cCtx.Int64(generationPauseDelay))) v.SetWorkerSleepDelayMillis(time.Duration(cCtx.Int64(workerSleepDelay))) + v.SetPprofInterval(cCtx.String(pprofInterval)) partitionSizeMB := cCtx.Uint64(partitionSizeMB) if partitionSizeMB != 0 { @@ -256,35 +259,3 @@ func expandCommaSeparators(in []string) []string { } return ret } - -func startPeriodicHeapProfileCollection(ctx context.Context) { - go func() { - ticker := time.NewTicker(10 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - collectHeapUsage() - } - } - }() - -} - -func collectHeapUsage() { - heapFileName := fmt.Sprintf("heap-%s.out", time.Now().UTC().Format("20060102T150405Z")) - heapFile, err := os.Create(heapFileName) - defer heapFile.Close() - - if err != nil { - panic(err) - } - - err = pprof.Lookup("heap").WriteTo(heapFile, 0) - if err != nil { - heapFile.WriteString(err.Error()) - } -} From de64f62140db1f477c656fbcacd4d18dda3f17b1 Mon Sep 17 00:00:00 2001 From: Jian Guan <61915096+tdq45gj@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:58:44 -0400 Subject: [PATCH 4/4] fix --- internal/verifier/pprof.go | 2 +- main/migration_verifier.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/verifier/pprof.go b/internal/verifier/pprof.go index f0876392..1ffc3371 100644 --- a/internal/verifier/pprof.go +++ b/internal/verifier/pprof.go @@ -40,6 +40,6 @@ func collectHeapUsage() { err = pprof.Lookup("heap").WriteTo(heapFile, 0) if err != nil { - heapFile.WriteString(err.Error()) + panic(err) } } diff --git a/main/migration_verifier.go b/main/migration_verifier.go index 787218fd..4289eb5d 100644 --- a/main/migration_verifier.go +++ b/main/migration_verifier.go @@ -149,7 +149,7 @@ func main() { }), altsrc.NewStringFlag(cli.StringFlag{ Name: pprofInterval, - Usage: "Interval to periodically collect pprof profiles", + Usage: "Interval to periodically collect pprof profiles (e.g. --pprofInterval=\"5m\")", }), }