Skip to content
Permalink
Browse files

Reduce memory usage of go/analysis

  • Loading branch information
jirfag committed Sep 21, 2019
1 parent 358ce7c commit 3aade55e059a82429be2eba7d3a03a14ce0a979f
@@ -560,6 +560,7 @@ Global Flags:
-j, --concurrency int Concurrency (default NumCPU) (default 8)
--cpu-profile-path string Path to CPU profile output file
--mem-profile-path string Path to memory profile output file
--trace-path string Path to trace output file
-v, --verbose verbose output
```
@@ -5,6 +5,7 @@ import (
"os"
"runtime"
"runtime/pprof"
"runtime/trace"
"strconv"

"github.com/spf13/cobra"
@@ -37,6 +38,16 @@ func (e *Executor) persistentPreRun(_ *cobra.Command, _ []string) {
runtime.MemProfileRate, _ = strconv.Atoi(rate)
}
}

if e.cfg.Run.TracePath != "" {
f, err := os.Create(e.cfg.Run.TracePath)
if err != nil {
e.log.Fatalf("Can't create file %s: %s", e.cfg.Run.TracePath, err)
}
if err = trace.Start(f); err != nil {
e.log.Fatalf("Can't start tracing: %s", err)
}
}
}

func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
@@ -58,6 +69,9 @@ func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
}
f.Close()
}
if e.cfg.Run.TracePath != "" {
trace.Stop()
}

os.Exit(e.exitCode)
}
@@ -136,6 +150,7 @@ func initRootFlagSet(fs *pflag.FlagSet, cfg *config.Config, needVersionOption bo

fs.StringVar(&cfg.Run.CPUProfilePath, "cpu-profile-path", "", wh("Path to CPU profile output file"))
fs.StringVar(&cfg.Run.MemProfilePath, "mem-profile-path", "", wh("Path to memory profile output file"))
fs.StringVar(&cfg.Run.TracePath, "trace-path", "", wh("Path to trace output file"))
fs.IntVarP(&cfg.Run.Concurrency, "concurrency", "j", getDefaultConcurrency(), wh("Concurrency (default NumCPU)"))
if needVersionOption {
fs.BoolVar(&cfg.Run.PrintVersion, "version", false, wh("Print version"))
@@ -451,21 +451,30 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
startedAt := time.Now()
debugf("Started tracking time")

var rssValues []uint64
ticker := time.NewTicker(10 * time.Millisecond)
var maxRSSMB, totalRSSMB float64
var iterationsCount int
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()

logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1"
const MB = 1024 * 1024

track := func() {
debugf("Starting memory tracing iteration ...")
var m runtime.MemStats
runtime.ReadMemStats(&m)

if logEveryRecord {
debugf("Stopping memory tracing iteration, printing ...")
printMemStats(&m, logger)
}

rssValues = append(rssValues, m.Sys)
rssMB := float64(m.Sys) / MB
if rssMB > maxRSSMB {
maxRSSMB = rssMB
}
totalRSSMB += rssMB
iterationsCount++
}

for {
@@ -476,7 +485,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
case <-ctx.Done():
stop = true
debugf("Stopped resources tracking")
case <-ticker.C: // track every second
case <-ticker.C:
}

if stop {
@@ -485,19 +494,10 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
}
track()

var avg, max uint64
for _, v := range rssValues {
avg += v
if v > max {
max = v
}
}
avg /= uint64(len(rssValues))
avgRSSMB := totalRSSMB / float64(iterationsCount)

const MB = 1024 * 1024
maxMB := float64(max) / MB
logger.Infof("Memory: %d samples, avg is %.1fMB, max is %.1fMB",
len(rssValues), float64(avg)/MB, maxMB)
iterationsCount, avgRSSMB, maxRSSMB)
logger.Infof("Execution took %s", time.Since(startedAt))
close(done)
}
@@ -102,6 +102,7 @@ type Run struct {
Silent bool
CPUProfilePath string
MemProfilePath string
TracePath string
Concurrency int
PrintResourcesUsage bool `mapstructure:"print-resources-usage"`

@@ -102,6 +102,10 @@ func (r *FileReader) validateConfig() error {
return errors.New("option run.memprofilepath in config isn't allowed")
}

if c.Run.TracePath != "" {
return errors.New("option run.tracepath in config isn't allowed")
}

if c.Run.IsVerbose {
return errors.New("can't set run.verbose option with config: only on command-line")
}
@@ -114,12 +114,10 @@ func (lnt Linter) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Is
return nil, errors.Wrap(err, "failed to configure analyzers")
}

runner := newRunner(lnt.name, lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard)
runner := newRunner(lnt.name, lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard, lintCtx.NeedWholeProgram)

diags, errs := runner.run(lnt.analyzers, lintCtx.Packages)
for i := 1; i < len(errs); i++ {
lintCtx.Log.Warnf("%s error: %s", lnt.Name(), errs[i])
}
// Don't print all errs: they can duplicate.
if len(errs) != 0 {
return nil, errs[0]
}
@@ -46,12 +46,10 @@ func (ml MetaLinter) Run(ctx context.Context, lintCtx *linter.Context) ([]result
allAnalyzers = append(allAnalyzers, linter.analyzers...)
}

runner := newRunner("metalinter", lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard)
runner := newRunner("metalinter", lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard, lintCtx.NeedWholeProgram)

diags, errs := runner.run(allAnalyzers, lintCtx.Packages)
for i := 1; i < len(errs); i++ {
lintCtx.Log.Warnf("go/analysis metalinter error: %s", errs[i])
}
// Don't print all errs: they can duplicate.
if len(errs) != 0 {
return nil, errs[0]
}

0 comments on commit 3aade55

Please sign in to comment.
You can’t perform that action at this time.