generated from honeycombio/.github
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
## Which problem is this PR solving? More visibility ## Short description of the changes - Adds debug service that allows you to use pprof to visualize and analyze profiling data, listening on port 6060 - Also includes pyroscope deltaprof for delta profiles per Liz's comment on Refinery "On long-lived services like Refinery with large process uptimes, the number of distinct stacks and code addresses grows in the profile as time goes on; this causes high numbers of bytes to be scraped when collecting profiles for timeseries that aren't increasing in value very often but have been seen at least once in process lifetime." - Adjusts debug logging to use `LOG_LEVEL=DEBUG` to separate from this `DEBUG=true` to avoid all the debug logging and allow these to work independently of each other ## How to verify that this has the expected result set `DEBUG=true` and get profiling data
- Loading branch information
1 parent
e9d70a4
commit 47a2d9d
Showing
6 changed files
with
210 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,186 @@ | ||
| package debug | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "html/template" | ||
| "net" | ||
| "net/http" | ||
| "net/http/pprof" | ||
| "os" | ||
| "runtime" | ||
| "strconv" | ||
| "sync" | ||
| "syscall" | ||
|
|
||
| "github.com/honeycombio/ebpf-agent/config" | ||
| deltaprof "github.com/pyroscope-io/godeltaprof/http/pprof" | ||
| metrics "github.com/rcrowley/go-metrics" | ||
| "github.com/rcrowley/go-metrics/exp" | ||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| const addr = "localhost:6060" | ||
|
|
||
| // injectable debug service | ||
| type DebugService struct { | ||
| mux *http.ServeMux | ||
| urls []string | ||
| expVars map[string]interface{} | ||
| mutex sync.RWMutex | ||
| Config config.Config | ||
| } | ||
|
|
||
| func (s *DebugService) Start() error { | ||
| // Enables 1% mutex profiling, and 1s block profiling | ||
| // Values from github.com/DataDog/go-profiler-notes/blob/main/block.md#usage | ||
| runtime.SetBlockProfileRate(1000000) | ||
| runtime.SetMutexProfileFraction(100) | ||
|
|
||
| s.expVars = make(map[string]interface{}) | ||
|
|
||
| s.mux = http.NewServeMux() | ||
|
|
||
| // Add to the mux but don't add an index entry. | ||
| s.mux.HandleFunc("/", s.indexHandler) | ||
|
|
||
| s.HandleFunc("/debug/pprof/", pprof.Index) | ||
| s.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) | ||
| s.HandleFunc("/debug/pprof/profile", pprof.Profile) | ||
| s.HandleFunc("/debug/pprof/symbol", pprof.Symbol) | ||
| s.HandleFunc("/debug/pprof/trace", pprof.Trace) | ||
|
|
||
| s.HandleFunc("/debug/pprof/delta_heap", deltaprof.Heap) | ||
| s.HandleFunc("/debug/pprof/delta_block", deltaprof.Block) | ||
| s.HandleFunc("/debug/pprof/delta_mutex", deltaprof.Mutex) | ||
|
|
||
| s.HandleFunc("/debug/vars", s.expvarHandler) | ||
| s.Handle("/debug/metrics", exp.ExpHandler(metrics.DefaultRegistry)) | ||
| s.Publish("cmdline", os.Args) | ||
| s.Publish("memstats", Func(memstats)) | ||
|
|
||
| go func() { | ||
| configAddr := s.Config.DebugAddr | ||
| if configAddr != "" { | ||
| host, portStr, _ := net.SplitHostPort(configAddr) | ||
| addr := net.JoinHostPort(host, portStr) | ||
| log.Info(). | ||
| Str("addr", addr). | ||
| Msg("Debug service listening") | ||
|
|
||
| err := http.ListenAndServe(addr, s.mux) | ||
| log.Debug(). | ||
| Err(err). | ||
| Msg("debug http server error") | ||
| } else { | ||
| // Prefer to listen on addr, but will try to bind to the next 9 ports | ||
| // in case you have multiple services running on the same host. | ||
| for i := 0; i < 10; i++ { | ||
| host, portStr, _ := net.SplitHostPort(addr) | ||
| port, _ := strconv.Atoi(portStr) | ||
| port += i | ||
| addr := net.JoinHostPort(host, fmt.Sprint(port)) | ||
|
|
||
| log.Info(). | ||
| Str("addr", addr). | ||
| Msg("Debug service listening") | ||
|
|
||
| err := http.ListenAndServe(addr, s.mux) | ||
| log.Debug(). | ||
| Err(err). | ||
| Msg("debug http server error") | ||
|
|
||
| if err, ok := err.(*net.OpError); ok { | ||
| if err, ok := err.Err.(*os.SyscallError); ok { | ||
| if err.Err == syscall.EADDRINUSE { | ||
| // address already in use, try another | ||
| continue | ||
| } | ||
| } | ||
| } | ||
| break | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Use Handle and HandleFunc to add new services on the internal debugging port. | ||
| func (s *DebugService) Handle(pattern string, handler http.Handler) { | ||
| s.mutex.Lock() | ||
| defer s.mutex.Unlock() | ||
|
|
||
| s.urls = append(s.urls, pattern) | ||
| s.mux.Handle(pattern, handler) | ||
| } | ||
|
|
||
| func (s *DebugService) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { | ||
| s.mutex.Lock() | ||
| defer s.mutex.Unlock() | ||
|
|
||
| s.urls = append(s.urls, pattern) | ||
| s.mux.HandleFunc(pattern, handler) | ||
| } | ||
|
|
||
| // Publish an expvar at /debug/vars, possibly using Func | ||
| func (s *DebugService) Publish(name string, v interface{}) { | ||
| s.mutex.Lock() | ||
| defer s.mutex.Unlock() | ||
| if _, existing := s.expVars[name]; existing { | ||
| log.Panic().Msg("Reuse of exported var name: " + name) | ||
| } | ||
| s.expVars[name] = v | ||
| } | ||
|
|
||
| func (s *DebugService) indexHandler(w http.ResponseWriter, req *http.Request) { | ||
| s.mutex.RLock() | ||
| defer s.mutex.RUnlock() | ||
|
|
||
| if err := indexTmpl.Execute(w, s.urls); err != nil { | ||
| log.Debug().Err(err).Msg("error rendering debug index") | ||
| } | ||
| } | ||
|
|
||
| var indexTmpl = template.Must(template.New("index").Parse(` | ||
| <html> | ||
| <head> | ||
| <title>Debug Index</title> | ||
| </head> | ||
| <body> | ||
| <h2>Index</h2> | ||
| <table> | ||
| {{range .}} | ||
| <tr><td><a href="{{.}}?debug=1">{{.}}</a> | ||
| {{end}} | ||
| </table> | ||
| </body> | ||
| </html> | ||
| `)) | ||
|
|
||
| func (s *DebugService) expvarHandler(w http.ResponseWriter, r *http.Request) { | ||
| s.mutex.RLock() | ||
| defer s.mutex.RUnlock() | ||
|
|
||
| w.Header().Set("Content-Type", "application/json; charset=utf-8") | ||
| values := make(map[string]interface{}, len(s.expVars)) | ||
| for k, v := range s.expVars { | ||
| if f, ok := v.(Func); ok { | ||
| v = f() | ||
| } | ||
| values[k] = v | ||
| } | ||
| b, err := json.MarshalIndent(values, "", " ") | ||
| if err != nil { | ||
| log.Debug().Err(err).Msg("error encoding expvars") | ||
| } | ||
| w.Write(b) | ||
| } | ||
|
|
||
| func memstats() interface{} { | ||
| stats := new(runtime.MemStats) | ||
| runtime.ReadMemStats(stats) | ||
| return *stats | ||
| } | ||
|
|
||
| type Func func() interface{} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters