feat(api): dump heap profile (#4630)

* feat(api): dump heap profile

set CDS_MAX_HEAP_SIZE in bytes

engine/api/api.go

@@ -1,12 +1,16 @@
 package api
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"os/signal"
+	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -662,7 +666,7 @@ func (a *API) Serve(ctx context.Context) error {
 		log.Error("unable to init api metrics: %v", err)
 	}
 
-	//Intialize notification package
+	// Intialize notification package
 	notification.Init(a.Config.URL.UI)
 
 	log.Info("Initializing Authentication driver...")
@@ -857,6 +861,23 @@ func (a *API) Serve(ctx context.Context) error {
 		return sdk.WrapError(err, "Cannot upsert cds version")
 	}
 
+	// Dump heap to objecstore on SIGINFO
+	siginfoChan := make(chan os.Signal, 1)
+	signal.Notify(siginfoChan, sdk.SIGINFO)
+	go func() {
+		<-siginfoChan
+		signal.Stop(siginfoChan)
+		var buffer = new(bytes.Buffer)
+		pprof.Lookup("heap").WriteTo(buffer, 1)
+		var heapProfile = heapProfile{uuid: sdk.UUID()}
+		s, err := a.SharedStorage.Store(heapProfile, ioutil.NopCloser(buffer))
+		if err != nil {
+			log.Error("unable to upload heap profile: %v", err)
+			return
+		}
+		log.Error("api> heap dump uploaded to %s", s)
+	}()
+
 	log.Info("Starting CDS API HTTP Server on %s:%d", a.Config.HTTP.Addr, a.Config.HTTP.Port)
 	if err := s.ListenAndServe(); err != nil {
 		return fmt.Errorf("Cannot start HTTP server: %v", err)
@@ -872,3 +893,17 @@ func (a *API) PanicDump() func(s string) (io.WriteCloser, error) {
 		return cache.NewWriteCloser(a.Cache, cache.Key("api", "panic_dump", s), panicDumpTTL), nil
 	}
 }
+
+type heapProfile struct {
+	uuid string
+}
+
+var _ objectstore.Object = new(heapProfile)
+
+func (p heapProfile) GetName() string {
+	return p.uuid
+}
+func (p heapProfile) GetPath() string {
+	hostname, _ := os.Hostname()
+	return fmt.Sprintf("api-heap-profile-%d-%s", time.Now().Unix(), hostname)
+}

engine/service/metrics.go

@@ -5,12 +5,16 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strconv"
+	"sync"
 	"time"
 
-	"github.com/ovh/cds/sdk"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
+
+	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/log"
 )
 
 func CommonMetricsView(ctx context.Context) []*view.View {
@@ -76,6 +80,13 @@ func CommonMetricsView(ctx context.Context) []*view.View {
 		hostname, _ := os.Hostname()
 		ctx, _ = tag.New(ctx, tag.Upsert(tagHostname, hostname))
 
+		var maxMemoryS = os.Getenv("CDS_MAX_HEAP_SIZE") // in bytes
+		var maxMemory uint64
+		var onceMaxMemorySignal = new(sync.Once)
+		if maxMemoryS != "" {
+			maxMemory, _ = strconv.ParseUint(maxMemoryS, 10, 64)
+		}
+
 		var tick = time.NewTicker(10 * time.Second)
 		defer tick.Stop()
 		for {
@@ -89,6 +100,21 @@ func CommonMetricsView(ctx context.Context) []*view.View {
 				stats.Record(ctx, totalAllocStats.M(int64(m.TotalAlloc)))
 				stats.Record(ctx, sysStats.M(int64(m.Sys)))
 				stats.Record(ctx, gcStats.M(int64(m.NumGC)))
+
+				if maxMemory > 0 && m.Alloc >= maxMemory {
+					onceMaxMemorySignal.Do(func() {
+						p, err := os.FindProcess(os.Getpid())
+						if err != nil {
+							log.Error("unable to find current process: %v", err)
+							return
+						}
+						if err := p.Signal(sdk.SIGINFO); err != nil {
+							log.Error("unable to send signal: %v", err)
+							return
+						}
+						log.Info("metrics> SIGINFO signal send to %v", os.Getpid())
+					})
+				}
 			}
 		}
 	})

go.mod

@@ -13,7 +13,6 @@ require (
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/SermoDigital/jose v0.9.1 // indirect
 	github.com/Shopify/sarama v1.19.0
-	github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect
 	github.com/alecthomas/jsonschema v0.0.0-20190429041900-eff3f6c90428
 	github.com/andygrunwald/go-gerrit v0.0.0-20181207071854-19ef3e9332a4
 	github.com/araddon/gou v0.0.0-20180315155215-820e9f87cd05 // indirect
@@ -59,8 +58,6 @@ require (
 	github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 // indirect
 	github.com/duosecurity/duo_api_golang v0.0.0-20180315112207-d0530c80e49a // indirect
 	github.com/eapache/go-resiliency v1.1.0
-	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v0.0.0-20161018201348-a360acfe359f
 	github.com/fatih/structs v1.0.0
 	github.com/fernet/fernet-go v0.0.0-20151007213151-1b2437bc582b // indirect
@@ -79,10 +76,8 @@ require (
 	github.com/go-sql-driver/mysql v0.0.0-20180113200744-2cc627ac8def // indirect
 	github.com/go-stomp/stomp v1.0.1 // indirect
 	github.com/gocql/gocql v0.0.0-20181018123354-22229812a83e // indirect
-	github.com/gogo/protobuf v1.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect
 	github.com/golang/protobuf v1.3.1
-	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect
 	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
 	github.com/google/gops v0.0.0-20170728214508-806455e841dc
@@ -167,8 +162,6 @@ require (
 	github.com/nwaples/rardecode v1.0.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.0-20160621093029-daf2955e742c
 	github.com/olivere/elastic v6.2.17+incompatible // indirect
-	github.com/onsi/ginkgo v1.7.0 // indirect
-	github.com/onsi/gomega v1.4.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.0.0-20180718062236-bc1467269fce // indirect
@@ -185,12 +178,9 @@ require (
 	github.com/pelletier/go-toml v0.0.0-20171222114548-0131db6d737c
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/phayes/permbits v0.0.0-20160816172359-32459f104210
-	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20170505125900-c90ca0c84f15
 	github.com/pkg/errors v0.8.0
 	github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6 // indirect
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20160620083229-6f4757563362
 	github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
@@ -240,7 +230,7 @@ require (
 	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
 	golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
-	golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 // indirect
+	golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82
 	golang.org/x/text v0.3.2
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
 	google.golang.org/grpc v1.20.1
@@ -277,3 +267,5 @@ replace github.com/docker/docker => github.com/docker/engine v0.0.0-201808160814
 replace github.com/ovh/cds/sdk/interpolate => ./sdk/interpolate
 
 replace github.com/ovh/cds/sdk/izanami => ./sdk/izanami
+
+go 1.13

sdk/signal_darwin.go

@@ -0,0 +1,5 @@
+package sdk
+
+import "golang.org/x/sys/unix"
+
+var SIGINFO = unix.SIGINFO

sdk/signal_linux.go

@@ -0,0 +1,5 @@
+package sdk
+
+import "golang.org/x/sys/unix"
+
+var SIGINFO = unix.SIGPWR

sdk/signal_windows.go

@@ -0,0 +1,5 @@
+package sdk
+
+import "golang.org/x/sys/unix"
+
+var SIGINFO = unix.SIGPWR