From d0a63dc1fcdbf7bf8793cbfb4e38cfe9e58daf5e Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 2 Mar 2022 17:22:13 +0100 Subject: [PATCH] Refactor debuginfo package Signed-off-by: Kemal Akkoyun --- cmd/debug-info/main.go | 22 +- go.mod | 2 + go.sum | 2 + pkg/debuginfo/debuginfo.go | 434 +++++++------------------------- pkg/debuginfo/debuginfo_file.go | 127 ---------- pkg/debuginfo/extract.go | 275 ++++++++++++++++++++ pkg/debuginfo/find.go | 119 +++++++++ pkg/debuginfo/upload.go | 101 ++++++++ pkg/profiler/profiler.go | 10 +- 9 files changed, 616 insertions(+), 476 deletions(-) delete mode 100644 pkg/debuginfo/debuginfo_file.go create mode 100644 pkg/debuginfo/extract.go create mode 100644 pkg/debuginfo/find.go create mode 100644 pkg/debuginfo/upload.go diff --git a/cmd/debug-info/main.go b/cmd/debug-info/main.go index 64c840e4e3..6372aaef0e 100644 --- a/cmd/debug-info/main.go +++ b/cmd/debug-info/main.go @@ -67,11 +67,7 @@ func main() { kongCtx := kong.Parse(&flags) logger := logger.NewLogger(flags.LogLevel, logger.LogFormatLogfmt, "") - - var ( - g run.Group - debugInfoClient = debuginfo.NewNoopClient() - ) + debugInfoClient := debuginfo.NewNoopClient() if len(flags.Upload.StoreAddress) > 0 { level.Debug(logger).Log("msg", "configuration", "bearertoken", flags.Upload.BearerToken, "insecure", flags.Upload.Insecure) @@ -86,7 +82,9 @@ func main() { } die := debuginfo.NewExtractor(logger, debugInfoClient, flags.TempDir) + diu := debuginfo.NewUploader(logger, debugInfoClient) + var g run.Group ctx, cancel := context.WithCancel(context.Background()) switch kongCtx.Command() { case "upload ": @@ -105,7 +103,17 @@ func main() { return errors.New("failed to find actionable files") } - return die.Upload(ctx, buildIDFiles) + debugInfoFiles, err := die.ExtractAll(ctx, buildIDFiles) + if err != nil { + return fmt.Errorf("failed to extract debug information: %w", err) + } + defer func() { + for _, f := range debugInfoFiles { + os.Remove(f) + } + }() + + return diu.UploadAll(ctx, debugInfoFiles) }, func(error) { cancel() }) @@ -131,7 +139,7 @@ func main() { return errors.New("failed to find actionable files") } - files, err := die.Extract(ctx, buildIDFiles) + files, err := die.ExtractAll(ctx, buildIDFiles) if err != nil { return err } diff --git a/go.mod b/go.mod index 170a4f3dc1..ace72ab712 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,8 @@ require ( github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index e444f81ecd..a1432f99fb 100644 --- a/go.sum +++ b/go.sum @@ -1063,6 +1063,7 @@ github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPA github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -1077,6 +1078,7 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= diff --git a/pkg/debuginfo/debuginfo.go b/pkg/debuginfo/debuginfo.go index 344b06fef5..63de23f369 100644 --- a/pkg/debuginfo/debuginfo.go +++ b/pkg/debuginfo/debuginfo.go @@ -14,26 +14,14 @@ package debuginfo import ( - "bytes" "context" - "debug/elf" "errors" - "fmt" "io" - "io/ioutil" "os" - "os/exec" - "path" - "strings" - "sync" - "time" - "github.com/cenkalti/backoff/v4" - "github.com/containerd/containerd/sys/reaper" "github.com/go-kit/log" "github.com/go-kit/log/level" lru "github.com/hashicorp/golang-lru" - "github.com/parca-dev/parca/pkg/symbol/elfutils" "github.com/parca-dev/parca-agent/pkg/objectfile" ) @@ -59,377 +47,149 @@ func NewNoopClient() Client { return &NoopClient{} } -type Extractor struct { +type DebugInfo struct { logger log.Logger + client Client - client Client - dbgFileCache *lru.ARCCache + existsCache *lru.ARCCache + debugInfoFileCache *lru.ARCCache - tmpDir string - - pool sync.Pool + *Extractor + *Uploader + *Finder } -// TODO(kakkoyun): Split extract and upload into separate layers. -// - Use debuginfo_file for extraction related operations. -func NewExtractor(logger log.Logger, client Client, tmpDir string) *Extractor { - cache, err := lru.NewARC(128) // Arbitrary cache size. +// New creates a new DebugInfo. +func New(logger log.Logger, client Client, tmp string) *DebugInfo { + ec, err := lru.NewARC(128) // Arbitrary cache size. + if err != nil { + level.Warn(logger).Log("msg", "failed to initialize exists cache", "err", err) + } + dc, err := lru.NewARC(128) // Arbitrary cache size. if err != nil { - level.Warn(logger).Log("msg", "failed to initialize debug file cache", "err", err) + level.Warn(logger).Log("msg", "failed to initialize debug info cache", "err", err) } - return &Extractor{ - logger: logger, - client: client, - tmpDir: tmpDir, - dbgFileCache: cache, - pool: sync.Pool{ - New: func() interface{} { - return bytes.NewBuffer(nil) - }, - }, + return &DebugInfo{ + logger: logger, + existsCache: ec, + debugInfoFileCache: dc, + client: client, + Extractor: NewExtractor(logger, client, tmp), + Uploader: NewUploader(logger, client), + Finder: NewFinder(logger), } } -func (di *Extractor) Upload(ctx context.Context, objFilePaths map[string]string) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } +// EnsureUploaded ensures that the extracted or the found debuginfo for the given buildID is uploaded. +func (di *DebugInfo) EnsureUploaded(ctx context.Context, objFiles []*objectfile.MappedObjectFile) { + for _, objFile := range objFiles { + buildID := objFile.BuildID - for buildID, filePath := range objFilePaths { - exists, err := di.client.Exists(ctx, buildID) - if err != nil { - level.Error(di.logger).Log("msg", "failed to check whether build ID symbol exists", "err", err) + if exists := di.exists(ctx, buildID, objFile.Path); exists { continue } - if !exists { - level.Debug(di.logger).Log("msg", "could not find symbols in server", "buildid", buildID) - - hasDebugInfo, err := checkIfFileHasDebugInfo(filePath) - if err != nil { - level.Debug(di.logger).Log( - "msg", "failed to determine whether file has debug symbols", - "file", filePath, "err", err, - ) - continue - } - - if !hasDebugInfo { - level.Debug(di.logger).Log( - "msg", "file does not have debug information, skipping", - "file", filePath, "err", err, - ) - continue - } - - debugInfoFile, err := di.extract(ctx, buildID, filePath) - if err != nil { - level.Debug(di.logger).Log( - "msg", "failed to extract debug information", - "buildid", buildID, "file", filePath, "err", err, - ) - continue - } - - if err := di.uploadDebugInfo(ctx, buildID, debugInfoFile); err != nil { - os.Remove(debugInfoFile) - level.Error(di.logger).Log( - "msg", "failed to upload debug information", - "buildid", buildID, "file", filePath, "err", err, - ) - continue - } - - os.Remove(debugInfoFile) - level.Info(di.logger).Log( - "msg", "debug information uploaded successfully", - "buildid", buildID, "file", filePath, + // Finds the debuginfo file. Interim files can be clean up. + dbgInfoPath, shouldCleanup := di.debugInfoFilePath(ctx, buildID, objFile) + // If debuginfo file is still not found, we don't need to upload anything. + if dbgInfoPath == "" { + level.Warn(di.logger).Log( + "msg", "failed to find debug info", + "buildid", objFile.BuildID, "path", objFile.Path, ) continue } - level.Info(di.logger).Log("msg", "debug information already exist in server", "buildid", buildID) - } - - return nil -} - -func (di *Extractor) Extract(ctx context.Context, objFilePaths map[string]string) ([]string, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - files := []string{} - for buildID, filePath := range objFilePaths { - debugInfoFile, err := di.extract(ctx, buildID, filePath) - if err != nil { - level.Error(di.logger).Log( - "msg", "failed to extract debug information", "buildid", - buildID, "file", filePath, "err", err) + // If we found a debuginfo file, either in file or on the system, we upload it to the server. + if err := di.Upload(ctx, objFile.BuildID, dbgInfoPath); err != nil { + level.Error(di.logger).Log("msg", "failed to upload debug info", "err", err) continue } - files = append(files, debugInfoFile) - } - - return files, nil -} -func (di *Extractor) EnsureUploaded(ctx context.Context, objFiles []*objectfile.MappedObjectFile) { - for _, objFile := range objFiles { - buildID := objFile.BuildID - exists, err := di.client.Exists(ctx, buildID) - if err != nil { - level.Warn(di.logger).Log( - "msg", "failed to check whether build ID symbol exists", - "buildid", buildID, "err", err, - ) - continue - } + level.Debug(di.logger).Log( + "msg", "debug info uploaded successfully", + "buildid", objFile.BuildID, "path", objFile.Path, + ) - if !exists { - level.Debug(di.logger).Log("msg", "could not find symbols in server", "buildid", buildID) - var dbgInfoFile *debugInfoFile - if di.dbgFileCache != nil { - if val, ok := di.dbgFileCache.Get(buildID); ok { - dbgInfoFile = val.(*debugInfoFile) - } else { - f, err := newDebugInfoFile(objFile) - if err != nil { - level.Debug(di.logger).Log( - "msg", "failed to create debug information file", - "buildid", buildID, "err", err, - ) - continue - } - di.dbgFileCache.Add(buildID, f) - dbgInfoFile = f - } - } - objFilePath := objFile.Path - if !dbgInfoFile.hasDebugInfo { - // The object does not have debug symbols, but maybe debuginfos - // have been installed separately, typically in /usr/lib/debug, so - // we try to discover if there is a debuginfo file, that has the - // same build ID as the object. - level.Debug(di.logger).Log( - "msg", "could not find symbols in binary, checking for additional debug info files on the system", - "buildid", objFile.BuildID, "file", objFilePath, - ) - if dbgInfoFile.localDebugInfoPath == "" { - // Binary does not have debug symbols, and we could not find any on the system. Nothing to do here. + // Successfully uploaded, we can clean up. + // Cleanup the extracted debug info file. + if shouldCleanup { + if err := os.Remove(dbgInfoPath); err != nil { + if os.IsNotExist(err) { + di.debugInfoFileCache.Remove(buildID) continue } - objFilePath = dbgInfoFile.localDebugInfoPath - } - - extractedDbgInfo, err := di.extract(ctx, buildID, objFilePath) - if err != nil { - level.Debug(di.logger).Log( - "msg", "failed to extract debug information", - "buildid", buildID, "file", objFilePath, "err", err, - ) - continue - } - - if err := di.uploadDebugInfo(ctx, buildID, extractedDbgInfo); err != nil { - os.Remove(extractedDbgInfo) - level.Warn(di.logger).Log(""+ - "msg", "failed to upload debug information", - "buildid", buildID, "file", objFilePath, "err", err, - ) + level.Debug(di.logger).Log("msg", "failed to cleanup debug info", "err", err) continue } - - os.Remove(extractedDbgInfo) - level.Debug(di.logger).Log( - "msg", "debug information uploaded successfully", - "buildid", buildID, "file", objFilePath, - ) - continue + di.debugInfoFileCache.Remove(buildID) } - - level.Debug(di.logger).Log( - "msg", "debug information already exist in server", - "buildid", buildID, - ) } } -func (di *Extractor) extract(ctx context.Context, buildID, file string) (string, error) { - tmpDir := path.Join(di.tmpDir, buildID) - if err := os.MkdirAll(tmpDir, 0o755); err != nil { - return "", fmt.Errorf("failed to create temp dir for debug information extraction: %w", err) - } +func (di *DebugInfo) exists(ctx context.Context, buildID, filePath string) bool { + // TODO(kakkoyun): Enable. + //if _, ok := di.existsCache.Get(buildID); ok { + // level.Debug(di.logger).Log("msg", "debug info already uploaded to server", "buildid", buildID, "path", filePath) + // return true + //} - hasDWARF, err := elfutils.HasDWARF(file) + exists, err := di.client.Exists(ctx, buildID) if err != nil { - level.Debug(di.logger).Log("msg", "failed to determine if binary has DWARF sections", - "path", file, "err", err, + level.Debug(di.logger).Log( + "msg", "failed to check whether build ID symbol exists", + "buildid", buildID, "err", err, ) } - isGo, err := elfutils.IsSymbolizableGoObjFile(file) - if err != nil { - level.Debug(di.logger).Log("msg", "failed to determine if binary is a Go binary", "path", file, "err", err) - } - - toRemove, err := sectionsToRemove(file) - if err != nil { - level.Debug(di.logger).Log("msg", "failed to determine sections to remove", "path", file, "err", err) - } - - outFile := path.Join(tmpDir, "debuginfo") - interimDir, err := ioutil.TempDir(di.tmpDir, "*") - if err != nil { - return "", err - } - defer func() { os.RemoveAll(interimDir) }() - - var cmd *exec.Cmd - switch { - case hasDWARF: - cmd = di.strip(ctx, interimDir, file, outFile, toRemove) - case isGo: - cmd = di.objcopy(ctx, file, outFile, toRemove) - default: - cmd = di.strip(ctx, interimDir, file, outFile, toRemove) - } - const msg = "failed to extract debug information from binary" - if err := di.run(cmd); err != nil { - return "", fmt.Errorf(msg+": %w", err) - } - - // Check if the debug information file is actually created. - if exists, err := exists(outFile); !exists { - if err != nil { - return "", fmt.Errorf(msg+": %w", err) - } - return "", fmt.Errorf(msg+": %s", "debug information file is not created") + if exists { + level.Debug(di.logger).Log( + "msg", "debug information already exist in server", + "buildid", buildID, "path", filePath, + ) + di.existsCache.Add(buildID, struct{}{}) + return true } - return outFile, nil -} -func (di *Extractor) run(cmd *exec.Cmd) error { level.Debug(di.logger).Log( - "msg", "running external binary utility command", "cmd", - strings.Join(cmd.Args, " "), - ) - b := di.pool.Get().(*bytes.Buffer) - defer func() { - b.Reset() - di.pool.Put(b) - }() - cmd.Stdout = b - cmd.Stderr = b - c, err := reaper.Default.Start(cmd) - if err != nil { - return err - } - const msg = "external binary utility command failed" - status, err := reaper.Default.Wait(cmd, c) - if err != nil { - level.Debug(di.logger).Log("msg", msg, "cmd", cmd.Args, "output", b.String(), "err", err) - return err - } - if status != 0 { - level.Debug(di.logger).Log("msg", msg, "cmd", cmd.Args, "output", b.String()) - return errors.New(msg) - } - return nil -} - -func (di *Extractor) strip(ctx context.Context, tmpDir, file, outFile string, toRemove []string) *exec.Cmd { - level.Debug(di.logger).Log("msg", "using eu-strip", "file", file) - // Extract debug symbols. - // If we have DWARF symbols, they are enough for us to symbolize the profiles. - // We observed that having DWARF debug symbols and symbol table together caused us problem in certain cases. - // As DWARF symbols enough on their own we just extract those. - // eu-strip --strip-debug extracts the .debug/.zdebug sections from the object files. - args := []string{"--strip-debug"} - for _, s := range toRemove { - args = append(args, "--remove-section", s) - } - args = append(args, - "-f", outFile, - "-o", path.Join(tmpDir, "binary.stripped"), - file, + "msg", "could not find symbols in server", + "buildid", buildID, "path", filePath, ) - return exec.CommandContext(ctx, "eu-strip", args...) + return false } -func (di *Extractor) objcopy(ctx context.Context, file, outFile string, toRemove []string) *exec.Cmd { - level.Debug(di.logger).Log("msg", "using objcopy", "file", file) - // Go binaries has a special case. They use ".gopclntab" section to symbolize addresses. - // We need to keep ".note.go.buildid", ".symtab" and ".gopclntab", - // however it doesn't hurt to keep rather small sections. - args := []string{} - toRemove = append(toRemove, ".text", ".rodata*") - for _, s := range toRemove { - args = append(args, "--remove-section", s) +func (di *DebugInfo) debugInfoFilePath(ctx context.Context, buildID string, objFile *objectfile.MappedObjectFile) (string, bool) { + type result struct { + path string + shouldCleanup bool } - args = append(args, - file, // source - outFile, // destination - ) - return exec.CommandContext(ctx, "objcopy", args...) -} + // TODO(kakkoyun): Enable. + //if val, ok := di.debugInfoFileCache.Get(buildID); ok { + // res := val.(result) + // return res.path, res.shouldCleanup + //} -func (di *Extractor) uploadDebugInfo(ctx context.Context, buildID, filePath string) error { - f, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open temp file for debug information: %w", err) - } - - expBackOff := backoff.NewExponentialBackOff() - expBackOff.InitialInterval = time.Second - expBackOff.MaxElapsedTime = time.Minute - - err = backoff.Retry(func() error { - if _, err := di.client.Upload(ctx, buildID, f); err != nil { - di.logger.Log( - "msg", "failed to upload debug information", - "buildid", buildID, - "path", filePath, - "retry", expBackOff.NextBackOff(), - "err", err, - ) - } - return err - }, expBackOff) - if err != nil { - return fmt.Errorf("failed to upload debug information: %w", err) + dbgInfoPath, err := di.Extract(ctx, buildID, objFile.Path) + if err == nil && dbgInfoPath != "" { + di.debugInfoFileCache.Add(buildID, result{dbgInfoPath, true}) + return dbgInfoPath, true } - return nil -} - -func exists(path string) (bool, error) { - _, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil -} + // err != nil || dbgInfoPath == "" + level.Debug(di.logger).Log( + "msg", "failed to extract debug information", + "buildid", buildID, "path", objFile.Path, "err", err, + ) -func sectionsToRemove(path string) ([]string, error) { - var sections []string - f, err := elf.Open(path) + // The object does not have debug symbols, but maybe debuginfos + // have been installed separately, typically in /usr/lib/debug, so + // we try to discover if there is a debuginfo file, that has the + // same build ID as the object. + dbgInfoPath, err = di.Find(ctx, objFile.BuildID, objFile.Root()) if err != nil { - return sections, fmt.Errorf("failed to open elf file: %w", err) - } - defer f.Close() - - for _, sec := range f.Sections { - if dwarfSuffix(sec) != "" && sec.Type == elf.SHT_NOBITS { // causes some trouble when it's set to SHT_NOBITS - sections = append(sections, sec.Name) - } + level.Warn(di.logger).Log("msg", "failed to find debug info on the system", "err", err) } - return sections, nil + // Even if finder returns empty string, it doesn't matter extractor already failed. + di.debugInfoFileCache.Add(buildID, result{path: dbgInfoPath, shouldCleanup: false}) + return dbgInfoPath, false } diff --git a/pkg/debuginfo/debuginfo_file.go b/pkg/debuginfo/debuginfo_file.go deleted file mode 100644 index db1a3604b3..0000000000 --- a/pkg/debuginfo/debuginfo_file.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2022 The Parca Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package debuginfo - -import ( - "debug/elf" - "errors" - "fmt" - "os" - "path" - "path/filepath" - "strings" - - "github.com/parca-dev/parca-agent/pkg/buildid" - "github.com/parca-dev/parca-agent/pkg/objectfile" -) - -var dwarfSuffix = func(s *elf.Section) string { - switch { - case strings.HasPrefix(s.Name, ".debug_"): - return s.Name[7:] - case strings.HasPrefix(s.Name, ".zdebug_"): - return s.Name[8:] - case strings.HasPrefix(s.Name, "__debug_"): // macos - return s.Name[8:] - default: - return "" - } -} - -// TODO(kakkoyun): Use to keep track of state of uploaded files. -// - https://github.com/parca-dev/parca-agent/issues/256 -type debugInfoFile struct { - *objectfile.ObjectFile - - hasDebugInfo bool - localDebugInfoPath string -} - -func newDebugInfoFile(file *objectfile.MappedObjectFile) (*debugInfoFile, error) { - ldbg, err := checkIfHostHasLocalDebugInfo(file) - if err != nil { - if !errors.Is(err, errNotFound) { - return nil, fmt.Errorf("failed to check if host has local debug info: %w", err) - } - // Failed to find local debug info, so make sure it's empty path. - ldbg = "" - } - - hdbg, err := checkIfFileHasDebugInfo(file.Path) - if err != nil { - return nil, fmt.Errorf("failed to check if file has debug info: %w", err) - } - - return &debugInfoFile{ObjectFile: file.ObjectFile, localDebugInfoPath: ldbg, hasDebugInfo: hdbg}, nil -} - -func checkIfHostHasLocalDebugInfo(f *objectfile.MappedObjectFile) (string, error) { - var ( - found = false - file string - ) - // TODO(kakkoyun): Distros may have different locations for debuginfo files. - // Add support for all of them. - err := filepath.Walk(path.Join(f.Root(), "/usr/lib/debug"), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - buildID, err := buildid.BuildID(path) - if err != nil { - return fmt.Errorf("failed to extract elf build ID, %w", err) - } - if strings.EqualFold(buildID, f.BuildID) { - found = true - file = path - } - } - return nil - }) - if err != nil { - if os.IsNotExist(err) { - return "", errNotFound - } - - return "", fmt.Errorf("failed to walk debug files: %w", err) - } - - if !found { - return "", errNotFound - } - return file, nil -} - -func checkIfFileHasDebugInfo(filePath string) (bool, error) { - ef, err := elf.Open(filePath) - if err != nil { - return false, fmt.Errorf("failed to open elf: %w", err) - } - defer ef.Close() - - for _, section := range ef.Sections { - if checkIfSectionHasSymbols(section) { - return true, nil - } - } - return false, nil -} - -func checkIfSectionHasSymbols(section *elf.Section) bool { - return section.Type == elf.SHT_SYMTAB || - strings.HasPrefix(section.Name, ".debug_") || - strings.HasPrefix(section.Name, ".zdebug_") || - strings.HasPrefix(section.Name, "__debug_") || // macos - section.Name == ".gopclntab" // go -} diff --git a/pkg/debuginfo/extract.go b/pkg/debuginfo/extract.go new file mode 100644 index 0000000000..a1d6315ea2 --- /dev/null +++ b/pkg/debuginfo/extract.go @@ -0,0 +1,275 @@ +// Copyright 2022 The Parca Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package debuginfo + +import ( + "bytes" + "context" + "debug/elf" + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "sync" + + "github.com/containerd/containerd/sys/reaper" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/hashicorp/go-multierror" + "github.com/parca-dev/parca/pkg/symbol/elfutils" +) + +// Extractor extracts debug information from a binary. +type Extractor struct { + logger log.Logger + client Client + pool sync.Pool + + tmpDir string +} + +// NewExtractor creates a new Extractor. +func NewExtractor(logger log.Logger, client Client, tmpDir string) *Extractor { + return &Extractor{ + logger: log.With(logger, "component", "extractor"), + client: client, + tmpDir: tmpDir, + pool: sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(nil) + }, + }, + } +} + +// ExtractAll extracts debug information from the given executables. +// It consumes a map of build id to executable path and returns a map of build id to extracted debug information path. +func (e *Extractor) ExtractAll(ctx context.Context, objFilePaths map[string]string) (map[string]string, error) { + files := map[string]string{} + var result *multierror.Error + for buildID, filePath := range objFilePaths { + debugInfoFile, err := e.Extract(ctx, buildID, filePath) + if err != nil { + level.Warn(e.logger).Log( + "msg", "failed to extract debug information", + "buildid", buildID, "file", filePath, "err", err, + ) + result = multierror.Append(result, err) + files[buildID] = "" + } + files[buildID] = debugInfoFile + } + + if len(result.Errors) == len(objFilePaths) { + return nil, result.ErrorOrNil() + } + return files, nil +} + +// Extract extracts debug information from the given executable. +// Cleaning up the temporary directory and the interim file is the caller's responsibility. +func (e *Extractor) Extract(ctx context.Context, buildID, filePath string) (string, error) { + select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + + tmpDir := path.Join(e.tmpDir, buildID) + if err := os.MkdirAll(tmpDir, 0o755); err != nil { + return "", fmt.Errorf("failed to create temp dir for debug information extraction: %w", err) + } + + hasSymtab, err := hasSymbols(filePath) + if err != nil { + level.Debug(e.logger).Log( + "msg", "failed to determine whether file has symbols", + "file", filePath, "err", err, + ) + } + + hasDWARF, err := elfutils.HasDWARF(filePath) + if err != nil { + level.Debug(e.logger).Log( + "msg", "failed to determine if binary has DWARF sections", + "path", filePath, "err", err, + ) + } + + isGo, err := elfutils.IsSymbolizableGoObjFile(filePath) + if err != nil { + level.Debug(e.logger).Log("msg", "failed to determine if binary is a Go binary", "path", filePath, "err", err) + } + + toRemove, err := sectionsToRemove(filePath) + if err != nil { + level.Debug(e.logger).Log("msg", "failed to determine sections to remove", "path", filePath, "err", err) + } + + outFile := path.Join(tmpDir, "debuginfo") + interimDir, err := ioutil.TempDir(e.tmpDir, "*") + if err != nil { + return "", err + } + defer func() { os.RemoveAll(interimDir) }() + + var cmd *exec.Cmd + switch { + case hasDWARF: + cmd = e.strip(ctx, interimDir, filePath, outFile, toRemove) + case isGo: + cmd = e.objcopy(ctx, filePath, outFile, toRemove) + case hasSymtab: + cmd = e.objcopy(ctx, filePath, outFile, toRemove) + default: + cmd = e.strip(ctx, interimDir, filePath, outFile, toRemove) + } + const msg = "failed to extract debug information from binary" + if err := e.run(cmd); err != nil { + return "", fmt.Errorf(msg+": %w", err) + } + + // Check if the debug information file is actually created. + if exists, err := exists(outFile); !exists { + if err != nil { + return "", fmt.Errorf(msg+": %w", err) + } + return "", fmt.Errorf(msg+": %s", "debug information file is not created") + } + + return outFile, nil +} + +func (e *Extractor) run(cmd *exec.Cmd) error { + level.Debug(e.logger).Log( + "msg", "running external binary utility command", "cmd", + strings.Join(cmd.Args, " "), + ) + b := e.pool.Get().(*bytes.Buffer) + defer func() { + b.Reset() + e.pool.Put(b) + }() + cmd.Stdout = b + cmd.Stderr = b + c, err := reaper.Default.Start(cmd) + if err != nil { + return err + } + const msg = "external binary utility command failed" + status, err := reaper.Default.Wait(cmd, c) + if err != nil { + level.Debug(e.logger).Log("msg", msg, "cmd", cmd.Args, "output", b.String(), "err", err) + return err + } + if status != 0 { + level.Debug(e.logger).Log("msg", msg, "cmd", cmd.Args, "output", b.String()) + return errors.New(msg) + } + return nil +} + +func (e *Extractor) strip(ctx context.Context, tmpDir, file, outFile string, toRemove []string) *exec.Cmd { + level.Debug(e.logger).Log("msg", "using eu-strip", "file", file) + // Extract debug symbols. + // If we have DWARF symbols, they are enough for us to symbolize the profiles. + // We observed that having DWARF debug symbols and symbol table together caused us problem in certain cases. + // As DWARF symbols enough on their own we just extract those. + // eu-strip --strip-debug extracts the .debug/.zdebug sections from the object files. + args := []string{"--strip-debug"} + for _, s := range toRemove { + args = append(args, "--remove-section", s) + } + args = append(args, + "-f", outFile, + "-o", path.Join(tmpDir, "binary.stripped"), + file, + ) + return exec.CommandContext(ctx, "eu-strip", args...) +} + +func (e *Extractor) objcopy(ctx context.Context, file, outFile string, toRemove []string) *exec.Cmd { + level.Debug(e.logger).Log("msg", "using objcopy", "file", file) + // Go binaries has a special case. They use ".gopclntab" section to symbolize addresses. + // We need to keep ".note.go.buildid", ".symtab" and ".gopclntab", + // however it doesn't hurt to keep rather small sections. + args := []string{} + toRemove = append(toRemove, ".text", ".rodata*") + for _, s := range toRemove { + args = append(args, "--remove-section", s) + } + args = append(args, + file, // source + outFile, // destination + ) + return exec.CommandContext(ctx, "objcopy", args...) +} + +var dwarfSuffix = func(s *elf.Section) string { + switch { + case strings.HasPrefix(s.Name, ".debug_"): + return s.Name[7:] + case strings.HasPrefix(s.Name, ".zdebug_"): + return s.Name[8:] + case strings.HasPrefix(s.Name, "__debug_"): // macos + return s.Name[8:] + default: + return "" + } +} + +func sectionsToRemove(path string) ([]string, error) { + var sections []string + f, err := elf.Open(path) + if err != nil { + return sections, fmt.Errorf("failed to open elf file: %w", err) + } + defer f.Close() + + for _, sec := range f.Sections { + if dwarfSuffix(sec) != "" && sec.Type == elf.SHT_NOBITS { // causes some trouble when it's set to SHT_NOBITS + sections = append(sections, sec.Name) + } + } + return sections, nil +} + +func exists(path string) (bool, error) { + _, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func hasSymbols(filePath string) (bool, error) { + ef, err := elf.Open(filePath) + if err != nil { + return false, fmt.Errorf("failed to open elf: %w", err) + } + defer ef.Close() + + for _, section := range ef.Sections { + if section.Type == elf.SHT_SYMTAB { + return true, nil + } + } + return false, nil +} diff --git a/pkg/debuginfo/find.go b/pkg/debuginfo/find.go new file mode 100644 index 0000000000..86284f42d0 --- /dev/null +++ b/pkg/debuginfo/find.go @@ -0,0 +1,119 @@ +// Copyright 2022 The Parca Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package debuginfo + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + lru "github.com/hashicorp/golang-lru" + + "github.com/parca-dev/parca-agent/pkg/buildid" +) + +// Finder finds the additional debug information on the system. +type Finder struct { + logger log.Logger + + cache *lru.ARCCache +} + +// NewFinder creates a new Finder. +func NewFinder(logger log.Logger) *Finder { + logger = log.With(logger, "component", "finder") + cache, err := lru.NewARC(128) // Arbitrary cache size. + if err != nil { + level.Warn(logger).Log("msg", "failed to initialize finder cache", "err", err) + } + return &Finder{ + logger: logger, + cache: cache, + } +} + +// Find finds the debug information for the given build ID. +func (f *Finder) Find(ctx context.Context, buildID, root string) (string, error) { + select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + + if val, ok := f.cache.Get(buildID); ok { + switch v := val.(type) { + case string: + return v, nil + case error: + return "", v + default: + // We didn't put you there?! + return "", errors.New("unexpected type") + } + } + + objFile, err := find(buildID, root) + if err != nil { + if errors.Is(err, errNotFound) { + f.cache.Add(buildID, err) + return "", err + } + } + + f.cache.Add(buildID, objFile) + return objFile, nil +} + +func find(buildID, root string) (string, error) { + var ( + found = false + file string + ) + // TODO(kakkoyun): Distros may have different locations for debuginfo files. + // Add support for all of them. + err := filepath.Walk(path.Join(root, "/usr/lib/debug"), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + id, err := buildid.BuildID(path) + if err != nil { + return fmt.Errorf("failed to extract elf build ID, %w", err) + } + if strings.EqualFold(id, buildID) { + found = true + file = path + } + } + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return "", errNotFound + } + + return "", fmt.Errorf("failed to walk debug files: %w", err) + } + + if !found { + return "", errNotFound + } + return file, nil +} diff --git a/pkg/debuginfo/upload.go b/pkg/debuginfo/upload.go new file mode 100644 index 0000000000..3a7543d662 --- /dev/null +++ b/pkg/debuginfo/upload.go @@ -0,0 +1,101 @@ +// Copyright 2022 The Parca Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package debuginfo + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/hashicorp/go-multierror" +) + +// Uploader uploads debug information to the Parca server. +type Uploader struct { + logger log.Logger + client Client +} + +// NewUploader creates a new Uploader. +func NewUploader(logger log.Logger, client Client) *Uploader { + return &Uploader{ + logger: log.With(logger, "component", "uploader"), + client: client, + } +} + +// UploadAll uploads all debug information to the Parca server. +func (u *Uploader) UploadAll(ctx context.Context, dbgInfoFilePaths map[string]string) error { + var result *multierror.Error + for buildID, filePath := range dbgInfoFilePaths { + if filePath == "" { + continue + } + if err := u.Upload(ctx, buildID, filePath); err != nil { + level.Warn(u.logger).Log( + "msg", "failed to upload debug information", + "buildid", buildID, "file", filePath, "err", err, + ) + result = multierror.Append(result, err) + continue + } + + level.Debug(u.logger).Log( + "msg", "debug info uploaded successfully", + "buildid", buildID, "file", filePath, + ) + } + + return result.ErrorOrNil() +} + +// Upload uploads the debug information to the Parca server. +func (u *Uploader) Upload(ctx context.Context, buildID, filePath string) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open temp file for debug information: %w", err) + } + + expBackOff := backoff.NewExponentialBackOff() + expBackOff.InitialInterval = time.Second + expBackOff.MaxElapsedTime = time.Minute + + err = backoff.Retry(func() error { + if _, err := u.client.Upload(ctx, buildID, f); err != nil { + level.Debug(u.logger).Log( + "msg", "failed to upload debug information", + "buildid", buildID, + "path", filePath, + "retry", expBackOff.NextBackOff(), + "err", err, + ) + } + return err + }, expBackOff) + if err != nil { + return fmt.Errorf("failed to upload debug information: %w", err) + } + + return nil +} diff --git a/pkg/profiler/profiler.go b/pkg/profiler/profiler.go index 98399e2e1b..bb6668277e 100644 --- a/pkg/profiler/profiler.go +++ b/pkg/profiler/profiler.go @@ -130,8 +130,8 @@ type CgroupProfiler struct { lastError error lastProfileTakenAt time.Time - writeClient profilestorepb.ProfileStoreServiceClient - debugInfoExtractor *debuginfo.Extractor + writeClient profilestorepb.ProfileStoreServiceClient + debugInfo *debuginfo.DebugInfo target model.LabelSet profilingDuration time.Duration @@ -159,8 +159,8 @@ func NewCgroupProfiler( pidMappingFileCache: maps.NewPIDMappingFileCache(logger), perfCache: perf.NewPerfCache(logger), objCache: objCache, - debugInfoExtractor: debuginfo.NewExtractor( - log.With(logger, "component", "debuginfoextractor"), + debugInfo: debuginfo.New( + log.With(logger, "component", "debuginfo"), debugInfoClient, tmp, ), @@ -524,7 +524,7 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time) } objFiles = append(objFiles, objFile) } - p.debugInfoExtractor.EnsureUploaded(ctx, objFiles) + p.debugInfo.EnsureUploaded(ctx, objFiles) }() // Resolve Kernel function names.