Skip to content

Commit

Permalink
Implements ability to change sample rate in go profiler
Browse files Browse the repository at this point in the history
Co-authored-by: alonlong <alonlong@163.com>
Co-authored-by: Dmitry Filimonov <dmitry@pyroscope.io>
  • Loading branch information
3 people committed May 25, 2021
1 parent 411e51e commit f98350d
Show file tree
Hide file tree
Showing 15 changed files with 1,590 additions and 34 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ lint:

.PHONY: ensure-logrus-not-used
ensure-logrus-not-used:
@! godepgraph -nostdlib -s ./pkg/agent/profiler/ | grep ' -> "github.com/sirupsen/logrus' \
@! go run "$(shell scripts/pinned-tool.sh github.com/kisielk/godepgraph)" -nostdlib -s ./pkg/agent/profiler/ | grep ' -> "github.com/sirupsen/logrus' \
|| (echo "\n^ ERROR: make sure ./pkg/agent/profiler/ does not depend on logrus. We don't want users' logs to be tainted. Talk to @petethepig if have questions\n" &1>2; exit 1)

.PHONY: unused
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
Expand Down Expand Up @@ -251,6 +252,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZOR
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand Down
46 changes: 39 additions & 7 deletions pkg/agent/gospy/gospy.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package gospy
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"runtime"
"runtime/pprof"
"sync"
"time"

custom_pprof "github.com/pyroscope-io/pyroscope/pkg/agent/pprof"
"github.com/pyroscope-io/pyroscope/pkg/agent/spy"
"github.com/pyroscope-io/pyroscope/pkg/convert"
)
Expand All @@ -22,22 +25,34 @@ type GoSpy struct {
stop bool
profileType spy.ProfileType
disableGCRuns bool
sampleRate uint32

lastGCGeneration uint32

stopCh chan struct{}
buf *bytes.Buffer
}

func Start(profileType spy.ProfileType, disableGCRuns bool) (spy.Spy, error) {
func startCPUProfile(w io.Writer, hz uint32) error {
// idea here is that for most people we're starting the default profiler
// but if you want to use a different sampling rate we use our experimental profiler
if hz == 100 {
return pprof.StartCPUProfile(w)
} else {
return custom_pprof.StartCPUProfile(w, hz)
}
}

func Start(profileType spy.ProfileType, sampleRate uint32, disableGCRuns bool) (spy.Spy, error) {
s := &GoSpy{
stopCh: make(chan struct{}),
buf: &bytes.Buffer{},
profileType: profileType,
disableGCRuns: disableGCRuns,
sampleRate: sampleRate,
}
if s.profileType == spy.ProfileCPU {
if err := pprof.StartCPUProfile(s.buf); err != nil {
if err := startCPUProfile(s.buf, sampleRate); err != nil {
return nil, err
}
}
Expand Down Expand Up @@ -84,21 +99,38 @@ func (s *GoSpy) Snapshot(cb func([]byte, uint64, error)) {
s.resetMutex.Lock()
defer s.resetMutex.Unlock()

// before the upload rate is reached, no need to read the profile data
if !s.reset {
return
}

s.reset = false

// TODO: handle errors
if s.profileType == spy.ProfileCPU {
// stop the previous cycle of sample collection
pprof.StopCPUProfile()
r, _ := gzip.NewReader(bytes.NewReader(s.buf.Bytes()))
profile, _ := convert.ParsePprof(r)
defer func() {
// start a new cycle of sample collection
if err := startCPUProfile(s.buf, s.sampleRate); err != nil {
cb(nil, uint64(0), err)
}
}()

// new gzip reader with the read data in buffer
r, err := gzip.NewReader(bytes.NewReader(s.buf.Bytes()))
if err != nil {
cb(nil, uint64(0), fmt.Errorf("new gzip reader: %v", err))
return
}

// parse the read data with pprof format
profile, err := convert.ParsePprof(r)
if err != nil {
cb(nil, uint64(0), fmt.Errorf("parse pprof: %v", err))
return
}
profile.Get("samples", func(name []byte, val int) {
cb(name, uint64(val), nil)
})
_ = pprof.StartCPUProfile(s.buf)
} else {
// this is current GC generation
currentGCGeneration := numGC()
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/gospy/gospy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ var _ = Describe("analytics", func() {
testing.WithConfig(func(cfg **config.Config) {
Describe("NewSession", func() {
It("works as expected", func(done Done) {
s, err := Start(spy.ProfileCPU, false)
s, err := Start(spy.ProfileCPU, 100, false)
Expect(err).ToNot(HaveOccurred())
go func() {
s := time.Now()
Expand Down
27 changes: 27 additions & 0 deletions pkg/agent/pprof/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
4 changes: 4 additions & 0 deletions pkg/agent/pprof/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
This is a copy of golang `runtime/pprof` package. The original code is available [here](https://github.com/golang/go/tree/master/src/runtime/pprof) and all the credit goes to the Go Authors.

It is licensed under BSD-style license found in the LICENSE file.

109 changes: 109 additions & 0 deletions pkg/agent/pprof/elf.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pprof

import (
"encoding/binary"
"errors"
"fmt"
"os"
)

var (
errBadELF = errors.New("malformed ELF binary")
errNoBuildID = errors.New("no NT_GNU_BUILD_ID found in ELF binary")
)

// elfBuildID returns the GNU build ID of the named ELF binary,
// without introducing a dependency on debug/elf and its dependencies.
func elfBuildID(file string) (string, error) {
buf := make([]byte, 256)
f, err := os.Open(file)
if err != nil {
return "", err
}
defer f.Close()

if _, err := f.ReadAt(buf[:64], 0); err != nil {
return "", err
}

// ELF file begins with \x7F E L F.
if buf[0] != 0x7F || buf[1] != 'E' || buf[2] != 'L' || buf[3] != 'F' {
return "", errBadELF
}

var byteOrder binary.ByteOrder
switch buf[5] {
default:
return "", errBadELF
case 1: // little-endian
byteOrder = binary.LittleEndian
case 2: // big-endian
byteOrder = binary.BigEndian
}

var shnum int
var shoff, shentsize int64
switch buf[4] {
default:
return "", errBadELF
case 1: // 32-bit file header
shoff = int64(byteOrder.Uint32(buf[32:]))
shentsize = int64(byteOrder.Uint16(buf[46:]))
if shentsize != 40 {
return "", errBadELF
}
shnum = int(byteOrder.Uint16(buf[48:]))
case 2: // 64-bit file header
shoff = int64(byteOrder.Uint64(buf[40:]))
shentsize = int64(byteOrder.Uint16(buf[58:]))
if shentsize != 64 {
return "", errBadELF
}
shnum = int(byteOrder.Uint16(buf[60:]))
}

for i := 0; i < shnum; i++ {
if _, err := f.ReadAt(buf[:shentsize], shoff+int64(i)*shentsize); err != nil {
return "", err
}
if typ := byteOrder.Uint32(buf[4:]); typ != 7 { // SHT_NOTE
continue
}
var off, size int64
if shentsize == 40 {
// 32-bit section header
off = int64(byteOrder.Uint32(buf[16:]))
size = int64(byteOrder.Uint32(buf[20:]))
} else {
// 64-bit section header
off = int64(byteOrder.Uint64(buf[24:]))
size = int64(byteOrder.Uint64(buf[32:]))
}
size += off
for off < size {
if _, err := f.ReadAt(buf[:16], off); err != nil { // room for header + name GNU\x00
return "", err
}
nameSize := int(byteOrder.Uint32(buf[0:]))
descSize := int(byteOrder.Uint32(buf[4:]))
noteType := int(byteOrder.Uint32(buf[8:]))
descOff := off + int64(12+(nameSize+3)&^3)
off = descOff + int64((descSize+3)&^3)
if nameSize != 4 || noteType != 3 || buf[12] != 'G' || buf[13] != 'N' || buf[14] != 'U' || buf[15] != '\x00' { // want name GNU\x00 type 3 (NT_GNU_BUILD_ID)
continue
}
if descSize > len(buf) {
return "", errBadELF
}
if _, err := f.ReadAt(buf[:descSize], descOff); err != nil {
return "", err
}
return fmt.Sprintf("%x", buf[:descSize]), nil
}
}
return "", errNoBuildID
}
89 changes: 89 additions & 0 deletions pkg/agent/pprof/map.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pprof

import "unsafe"

// A profMap is a map from (stack, tag) to mapEntry.
// It grows without bound, but that's assumed to be OK.
type profMap struct {
hash map[uintptr]*profMapEntry
all *profMapEntry
last *profMapEntry
free []profMapEntry
freeStk []uintptr
}

// A profMapEntry is a single entry in the profMap.
type profMapEntry struct {
nextHash *profMapEntry // next in hash list
nextAll *profMapEntry // next in list of all entries
stk []uintptr
tag unsafe.Pointer
count int64
}

func (m *profMap) lookup(stk []uint64, tag unsafe.Pointer) *profMapEntry {
// Compute hash of (stk, tag).
h := uintptr(0)
for _, x := range stk {
h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1)))
h += uintptr(x) * 41
}
h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1)))
h += uintptr(tag) * 41

// Find entry if present.
var last *profMapEntry
Search:
for e := m.hash[h]; e != nil; last, e = e, e.nextHash {
if len(e.stk) != len(stk) || e.tag != tag {
continue
}
for j := range stk {
if e.stk[j] != uintptr(stk[j]) {
continue Search
}
}
// Move to front.
if last != nil {
last.nextHash = e.nextHash
e.nextHash = m.hash[h]
m.hash[h] = e
}
return e
}

// Add new entry.
if len(m.free) < 1 {
m.free = make([]profMapEntry, 128)
}
e := &m.free[0]
m.free = m.free[1:]
e.nextHash = m.hash[h]
e.tag = tag

if len(m.freeStk) < len(stk) {
m.freeStk = make([]uintptr, 1024)
}
e.stk = m.freeStk[:len(stk)]
m.freeStk = m.freeStk[len(stk):]

for j := range stk {
e.stk[j] = uintptr(stk[j])
}
if m.hash == nil {
m.hash = make(map[uintptr]*profMapEntry)
}
m.hash[h] = e
if m.all == nil {
m.all = e
m.last = e
} else {
m.last.nextAll = e
m.last = e
}
return e
}

0 comments on commit f98350d

Please sign in to comment.