diff --git a/cmd/parca-agent/main.go b/cmd/parca-agent/main.go index 1e3fcaccaf..52f6485bc7 100644 --- a/cmd/parca-agent/main.go +++ b/cmd/parca-agent/main.go @@ -49,9 +49,7 @@ import ( "github.com/parca-dev/parca-agent/pkg/agent" "github.com/parca-dev/parca-agent/pkg/debuginfo" "github.com/parca-dev/parca-agent/pkg/discovery" - "github.com/parca-dev/parca-agent/pkg/ksym" "github.com/parca-dev/parca-agent/pkg/logger" - "github.com/parca-dev/parca-agent/pkg/objectfile" "github.com/parca-dev/parca-agent/pkg/target" "github.com/parca-dev/parca-agent/pkg/template" ) @@ -83,7 +81,7 @@ type flags struct { SystemdCgroupPath string `kong:"help='The cgroupfs path to a systemd slice.'"` } -func getExternalLabels(flagExternalLabels map[string]string, flagNode string) model.LabelSet { +func externalLabels(flagExternalLabels map[string]string, flagNode string) model.LabelSet { if flagExternalLabels == nil { flagExternalLabels = map[string]string{} } @@ -135,9 +133,6 @@ func main() { debugInfoClient = parcadebuginfo.NewDebugInfoClient(conn) } - ksymCache := ksym.NewKsymCache(logger) - objCache := objectfile.NewCache() - var ( configs discovery.Configs // TODO(Sylfrena): Make ticker duration configurable @@ -160,8 +155,13 @@ func main() { )) } - externalLabels := getExternalLabels(flags.ExternalLabel, flags.Node) - tm := target.NewManager(logger, reg, ksymCache, objCache, profileListener, debugInfoClient, flags.ProfilingDuration, externalLabels, flags.TempDir) + tm := target.NewManager( + logger, reg, + profileListener, debugInfoClient, + flags.ProfilingDuration, + externalLabels(flags.ExternalLabel, flags.Node), + flags.TempDir, + ) mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) mux.HandleFunc("/debug/pprof/", pprof.Index) @@ -233,7 +233,10 @@ func main() { err := template.StatusPageTemplate.Execute(w, statusPage) if err != nil { - http.Error(w, "Unexpected error occurred while rendering status page: "+err.Error(), http.StatusInternalServerError) + http.Error(w, + "Unexpected error occurred while rendering status page: "+err.Error(), + http.StatusInternalServerError, + ) } return @@ -244,7 +247,10 @@ func main() { query := r.URL.Query().Get("query") matchers, err := parser.ParseMetricSelector(query) if err != nil { - http.Error(w, `query incorrectly formatted, expecting selector in form of: {name1="value1",name2="value2"}`, http.StatusBadRequest) + http.Error(w, + `query incorrectly formatted, expecting selector in form of: {name1="value1",name2="value2"}`, + http.StatusBadRequest, + ) return } @@ -256,7 +262,12 @@ func main() { profile, err := profileListener.NextMatchingProfile(ctx, matchers) if profile == nil || err == context.Canceled { - http.Error(w, "No profile taken in the last 11 seconds that matches the requested label-matchers query. Profiles are taken every 10 seconds so either the profiler matching the label-set has stopped profiling, or the label-set was incorrect.", http.StatusNotFound) + http.Error(w, + "No profile taken in the last 11 seconds that matches the requested label-matchers query. "+ + "Profiles are taken every 10 seconds so either the profiler matching the label-set has stopped profiling, "+ + "or the label-set was incorrect.", + http.StatusNotFound, + ) return } if err != nil { diff --git a/go.mod b/go.mod index 89716933c7..170a4f3dc1 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect diff --git a/pkg/debuginfo/cache.go b/pkg/debuginfo/cache.go deleted file mode 100644 index 457f00ca94..0000000000 --- a/pkg/debuginfo/cache.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 The Parca Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package debuginfo - -import ( - "github.com/parca-dev/parca-agent/pkg/objectfile" -) - -type cache struct { - files map[string]*debugInfoFile -} - -func newCache() *cache { - return &cache{ - files: make(map[string]*debugInfoFile), - } -} - -func (c *cache) debugInfoFile(objFile *objectfile.ObjectFile) *debugInfoFile { - if dbgFile, ok := c.files[objFile.BuildID]; ok { - return dbgFile - } - dbgInfoFile := debugInfoFile{ObjectFile: objFile} - c.files[objFile.BuildID] = &dbgInfoFile - return &dbgInfoFile -} diff --git a/pkg/debuginfo/debuginfo.go b/pkg/debuginfo/debuginfo.go index b3d1fc6afa..14f7ef84c1 100644 --- a/pkg/debuginfo/debuginfo.go +++ b/pkg/debuginfo/debuginfo.go @@ -32,6 +32,7 @@ import ( "github.com/containerd/containerd/sys/reaper" "github.com/go-kit/log" "github.com/go-kit/log/level" + lru "github.com/hashicorp/golang-lru" "github.com/parca-dev/parca/pkg/symbol/elfutils" "github.com/parca-dev/parca-agent/pkg/objectfile" @@ -59,20 +60,28 @@ func NewNoopClient() Client { } type Extractor struct { - logger log.Logger - client Client - dbgCache *cache - tmpDir string + logger log.Logger + + client Client + dbgFileCache *lru.ARCCache + + tmpDir string pool sync.Pool } +// TODO(kakkoyun): Split extract and upload into separate layers. +// - Use debuginfo_file for extraction related operations. func NewExtractor(logger log.Logger, client Client, tmpDir string) *Extractor { + cache, err := lru.NewARC(128) // Arbitrary cache size. + if err != nil { + level.Warn(logger).Log("msg", "failed to initialize debug file cache", "err", err) + } return &Extractor{ - logger: logger, - client: client, - tmpDir: tmpDir, - dbgCache: newCache(), + logger: logger, + client: client, + tmpDir: tmpDir, + dbgFileCache: cache, pool: sync.Pool{ New: func() interface{} { return bytes.NewBuffer(nil) @@ -88,7 +97,7 @@ func (di *Extractor) Upload(ctx context.Context, objFilePaths map[string]string) default: } - for buildID, path := range objFilePaths { + for buildID, pth := range objFilePaths { exists, err := di.client.Exists(ctx, buildID) if err != nil { level.Error(di.logger).Log("msg", "failed to check whether build ID symbol exists", "err", err) @@ -98,31 +107,31 @@ func (di *Extractor) Upload(ctx context.Context, objFilePaths map[string]string) if !exists { level.Debug(di.logger).Log("msg", "could not find symbols in server", "buildid", buildID) - hasDebugInfo, err := checkDebugInfo(path) + hasDebugInfo, err := checkIfFileHasDebugInfo(pth) if err != nil { - level.Debug(di.logger).Log("msg", "failed to determine whether file has debug symbols", "file", path, "err", err) + level.Debug(di.logger).Log("msg", "failed to determine whether file has debug symbols", "file", pth, "err", err) continue } if !hasDebugInfo { - level.Debug(di.logger).Log("msg", "file does not have debug information, skipping", "file", path, "err", err) + level.Debug(di.logger).Log("msg", "file does not have debug information, skipping", "file", pth, "err", err) continue } - debugInfoFile, err := di.extract(ctx, buildID, path) + debugInfoFile, err := di.extract(ctx, buildID, pth) if err != nil { - level.Debug(di.logger).Log("msg", "failed to extract debug information", "buildid", buildID, "file", path, "err", err) + level.Debug(di.logger).Log("msg", "failed to extract debug information", "buildid", buildID, "file", pth, "err", err) continue } if err := di.uploadDebugInfo(ctx, buildID, debugInfoFile); err != nil { os.Remove(debugInfoFile) - level.Error(di.logger).Log("msg", "failed to upload debug information", "buildid", buildID, "file", path, "err", err) + level.Error(di.logger).Log("msg", "failed to upload debug information", "buildid", buildID, "file", pth, "err", err) continue } os.Remove(debugInfoFile) - level.Info(di.logger).Log("msg", "debug information uploaded successfully", "buildid", buildID, "file", path) + level.Info(di.logger).Log("msg", "debug information uploaded successfully", "buildid", buildID, "file", pth) continue } @@ -152,40 +161,46 @@ func (di *Extractor) Extract(ctx context.Context, objFilePaths map[string]string return files, nil } -func (di *Extractor) EnsureUploaded(ctx context.Context, objFiles []*objectfile.ObjectFile) { +func (di *Extractor) EnsureUploaded(ctx context.Context, objFiles []*objectfile.MappedObjectFile) { for _, objFile := range objFiles { buildID := objFile.BuildID - exists, err := di.client.Exists(ctx, objFile.BuildID) + exists, err := di.client.Exists(ctx, buildID) if err != nil { level.Warn(di.logger).Log("msg", "failed to check whether build ID symbol exists", "err", err) continue } if !exists { - level.Debug(di.logger).Log("msg", "could not find symbols in server", "buildid", objFile.BuildID) - dbgInfoFile := di.dbgCache.debugInfoFile(objFile) - objFilePath := objFile.FullPath() - hasDebugInfo, err := dbgInfoFile.HasDebugInfo() - if err != nil { - level.Debug(di.logger).Log("msg", "failed to determine whether file has debug symbols", "file", objFilePath, "err", err) - continue + level.Debug(di.logger).Log("msg", "could not find symbols in server", "buildid", buildID) + var dbgInfoFile *debugInfoFile + if di.dbgFileCache != nil { + if val, ok := di.dbgFileCache.Get(buildID); ok { + dbgInfoFile = val.(*debugInfoFile) + } else { + f, err := newDebugInfoFile(objFile) + if err != nil { + level.Debug(di.logger).Log("msg", "failed to create debug information file", "buildid", buildID, "err", err) + continue + } + di.dbgFileCache.Add(buildID, f) + dbgInfoFile = f + } } - - if !hasDebugInfo { + objFilePath := objFile.Path + if !dbgInfoFile.hasDebugInfo { // The object does not have debug symbols, but maybe debuginfos // have been installed separately, typically in /usr/lib/debug, so // we try to discover if there is a debuginfo file, that has the // same build ID as the object. - level.Debug(di.logger).Log("msg", "could not find symbols in binary, checking for additional debuginfo file", "buildid", objFile.BuildID, "file", objFilePath) - dbgInfo, err := dbgInfoFile.LocalHostDebugInfoPath() - if err != nil { - if !errors.Is(err, errNotFound) { - level.Debug(di.logger).Log("msg", "failed to find additional debug information", "root", objFile.Root(), "err", err) - } + level.Debug(di.logger).Log( + "msg", "could not find symbols in binary, checking for additional debug info files on the system", + "buildid", objFile.BuildID, "file", objFilePath, + ) + if dbgInfoFile.localDebugInfoPath == "" { + // Binary does not have debug symbols, and we could not find any on the system. Nothing to do here. continue } - - objFilePath = dbgInfo + objFilePath = dbgInfoFile.localDebugInfoPath } extractedDbgInfo, err := di.extract(ctx, buildID, objFilePath) diff --git a/pkg/debuginfo/debuginfo_file.go b/pkg/debuginfo/debuginfo_file.go index 21d3b17492..db1a3604b3 100644 --- a/pkg/debuginfo/debuginfo_file.go +++ b/pkg/debuginfo/debuginfo_file.go @@ -15,12 +15,12 @@ package debuginfo import ( "debug/elf" + "errors" "fmt" "os" "path" "path/filepath" "strings" - "sync" "github.com/parca-dev/parca-agent/pkg/buildid" "github.com/parca-dev/parca-agent/pkg/objectfile" @@ -39,26 +39,34 @@ var dwarfSuffix = func(s *elf.Section) string { } } +// TODO(kakkoyun): Use to keep track of state of uploaded files. +// - https://github.com/parca-dev/parca-agent/issues/256 type debugInfoFile struct { *objectfile.ObjectFile - dbgOnce sync.Once - dbgErr error - hasDebugInfo bool - - localDbgOnce sync.Once - localDbgErr error + hasDebugInfo bool localDebugInfoPath string } -func (f *debugInfoFile) LocalHostDebugInfoPath() (string, error) { - f.localDbgOnce.Do(func() { - f.localDebugInfoPath, f.localDbgErr = f.checkLocalHostDebugInfo() - }) - return f.localDebugInfoPath, f.localDbgErr +func newDebugInfoFile(file *objectfile.MappedObjectFile) (*debugInfoFile, error) { + ldbg, err := checkIfHostHasLocalDebugInfo(file) + if err != nil { + if !errors.Is(err, errNotFound) { + return nil, fmt.Errorf("failed to check if host has local debug info: %w", err) + } + // Failed to find local debug info, so make sure it's empty path. + ldbg = "" + } + + hdbg, err := checkIfFileHasDebugInfo(file.Path) + if err != nil { + return nil, fmt.Errorf("failed to check if file has debug info: %w", err) + } + + return &debugInfoFile{ObjectFile: file.ObjectFile, localDebugInfoPath: ldbg, hasDebugInfo: hdbg}, nil } -func (f *debugInfoFile) checkLocalHostDebugInfo() (string, error) { +func checkIfHostHasLocalDebugInfo(f *objectfile.MappedObjectFile) (string, error) { var ( found = false file string @@ -95,27 +103,25 @@ func (f *debugInfoFile) checkLocalHostDebugInfo() (string, error) { return file, nil } -func (f *debugInfoFile) HasDebugInfo() (bool, error) { - f.dbgOnce.Do(func() { f.hasDebugInfo, f.dbgErr = checkDebugInfo(f.FullPath()) }) - - return f.hasDebugInfo, f.dbgErr -} - -func checkDebugInfo(path string) (bool, error) { - ef, err := elf.Open(path) +func checkIfFileHasDebugInfo(filePath string) (bool, error) { + ef, err := elf.Open(filePath) if err != nil { return false, fmt.Errorf("failed to open elf: %w", err) } defer ef.Close() for _, section := range ef.Sections { - if section.Type == elf.SHT_SYMTAB || // TODO: Consider moving this to a specific func. - strings.HasPrefix(section.Name, ".debug_") || - strings.HasPrefix(section.Name, ".zdebug_") || - strings.HasPrefix(section.Name, "__debug_") || // macos - section.Name == ".gopclntab" { // go + if checkIfSectionHasSymbols(section) { return true, nil } } return false, nil } + +func checkIfSectionHasSymbols(section *elf.Section) bool { + return section.Type == elf.SHT_SYMTAB || + strings.HasPrefix(section.Name, ".debug_") || + strings.HasPrefix(section.Name, ".zdebug_") || + strings.HasPrefix(section.Name, "__debug_") || // macos + section.Name == ".gopclntab" // go +} diff --git a/pkg/maps/mapping.go b/pkg/maps/mapping.go index 4ed32221b3..8c5725543d 100644 --- a/pkg/maps/mapping.go +++ b/pkg/maps/mapping.go @@ -15,8 +15,6 @@ package maps import ( "github.com/google/pprof/profile" - - "github.com/parca-dev/parca-agent/pkg/objectfile" ) type Mapping struct { @@ -48,16 +46,23 @@ func (m *Mapping) PIDAddrMapping(pid uint32, addr uint64) (*profile.Mapping, err return mappingForAddr(maps, addr), nil } -func (m *Mapping) AllMappings() ([]*profile.Mapping, []*objectfile.ObjectFile) { +type ProcessMapping struct { + PID uint32 + Mapping *profile.Mapping +} + +func (m *Mapping) AllMappings() ([]*profile.Mapping, []ProcessMapping) { res := []*profile.Mapping{} - objFiles := []*objectfile.ObjectFile{} + mappedFiles := []ProcessMapping{} i := uint64(1) // Mapping IDs need to start with 1 in pprof. for _, pid := range m.pids { maps := m.pidMappings[pid] for _, mapping := range maps { if mapping.BuildID != "" { - // TODO(kakkoyun): Use objectfile.FromProcess() and add cache! - objFiles = append(objFiles, objectfile.NewObjectFile(pid, mapping)) + mappedFiles = append(mappedFiles, ProcessMapping{ + PID: pid, + Mapping: mapping, + }) } // TODO(brancz): Do we need to handle potentially duplicate // vdso/vsyscall mappings? @@ -67,7 +72,7 @@ func (m *Mapping) AllMappings() ([]*profile.Mapping, []*objectfile.ObjectFile) { } } - return res, objFiles + return res, mappedFiles } func mappingForAddr(mapping []*profile.Mapping, addr uint64) *profile.Mapping { diff --git a/pkg/objectfile/cache.go b/pkg/objectfile/cache.go index e5bdb56ef1..173ced1c0a 100644 --- a/pkg/objectfile/cache.go +++ b/pkg/objectfile/cache.go @@ -13,30 +13,64 @@ package objectfile -import "github.com/google/pprof/profile" +import ( + "fmt" + "path" + "strconv" -type Cache struct { - files map[string]*ObjectFile + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/google/pprof/profile" + lru "github.com/hashicorp/golang-lru" +) + +type Cache interface { + ObjectFileForProcess(pid uint32, m *profile.Mapping) (*MappedObjectFile, error) +} + +type cache struct { + cache *lru.ARCCache +} + +type noopCache struct{} + +func (n noopCache) ObjectFileForProcess(pid uint32, m *profile.Mapping) (*MappedObjectFile, error) { + return fromProcess(pid, m) } // NewCache creates a new cache for object files. -func NewCache() *Cache { - return &Cache{ - files: make(map[string]*ObjectFile), +func NewCache(logger log.Logger, size int) Cache { + c, err := lru.NewARC(size) + if err != nil { + level.Warn(logger).Log("msg", "failed to initialize cache", "err", err) + return &noopCache{} } + return &cache{cache: c} } // ObjectFileForProcess returns the object file for the given mapping and process id. // If object file is already in the cache, it is returned. // Otherwise, the object file is loaded from the file system. -func (c *Cache) ObjectFileForProcess(pid uint32, m *profile.Mapping) (*ObjectFile, error) { - if objFile, ok := c.files[m.BuildID]; ok { - return objFile, nil +func (c *cache) ObjectFileForProcess(pid uint32, m *profile.Mapping) (*MappedObjectFile, error) { + if val, ok := c.cache.Get(m.BuildID); ok { + return val.(*MappedObjectFile), nil } - objFile, err := FromProcess(pid, m) + + objFile, err := fromProcess(pid, m) if err != nil { return nil, err } - c.files[m.BuildID] = objFile + + c.cache.Add(m.BuildID, objFile) return objFile, nil } + +// fromProcess opens the specified executable or library file from the process. +func fromProcess(pid uint32, m *profile.Mapping) (*MappedObjectFile, error) { + filePath := path.Join("/proc", strconv.FormatUint(uint64(pid), 10), "/root", m.File) + objFile, err := Open(filePath, m) + if err != nil { + return nil, fmt.Errorf("failed to open mapped file: %v", err) + } + return &MappedObjectFile{ObjectFile: objFile, PID: pid, File: m.File}, nil +} diff --git a/pkg/objectfile/object_file.go b/pkg/objectfile/object_file.go index 991ab49a70..757a61adaf 100644 --- a/pkg/objectfile/object_file.go +++ b/pkg/objectfile/object_file.go @@ -34,17 +34,6 @@ import ( // Defined for testing. var elfOpen = elf.Open -// FromProcess opens the specified executable or library file from the process. -func FromProcess(pid uint32, m *profile.Mapping) (*ObjectFile, error) { - filePath := path.Join(fmt.Sprintf("/proc/%d/root", pid), m.File) - objFile, err := Open(filePath, m) - if err != nil { - return nil, err - } - objFile.PID = pid - return objFile, nil -} - // Open opens the specified executable or library file from the given path. func Open(filePath string, m *profile.Mapping) (*ObjectFile, error) { f, err := os.Open(filePath) @@ -129,8 +118,8 @@ func open(filePath string, start, limit, offset uint64, relocationSymbol string) return nil, fmt.Errorf("could not identify base for %s: %v", filePath, err) } return &ObjectFile{ + Path: filePath, BuildID: buildID, - path: filePath, m: &mapping{ start: start, limit: limit, @@ -140,26 +129,10 @@ func open(filePath string, start, limit, offset uint64, relocationSymbol string) }, nil } -func NewObjectFile(pid uint32, m *profile.Mapping) *ObjectFile { - return &ObjectFile{ - PID: pid, - File: m.File, - BuildID: m.BuildID, - m: &mapping{ - start: m.Start, - limit: m.Limit, - offset: m.Offset, - }, - } -} - type ObjectFile struct { - PID uint32 - File string + Path string BuildID string - path string - // Ensures the base, baseErr and isData are computed once. baseOnce sync.Once base uint64 @@ -169,15 +142,15 @@ type ObjectFile struct { m *mapping } -func (f *ObjectFile) Root() string { - return path.Join("/proc", strconv.FormatUint(uint64(f.PID), 10), "/root") +type MappedObjectFile struct { + *ObjectFile + + PID uint32 + File string } -func (f *ObjectFile) FullPath() string { - if f.path != "" { - return f.path - } - return path.Join(f.Root(), f.File) +func (f MappedObjectFile) Root() string { + return path.Join("/proc", strconv.FormatUint(uint64(f.PID), 10), "/root") } func (f *ObjectFile) ObjAddr(addr uint64) (uint64, error) { @@ -195,19 +168,18 @@ func (f *ObjectFile) computeBase(addr uint64) error { if f == nil || f.m == nil { return nil } - path := f.FullPath() if addr < f.m.start || addr >= f.m.limit { - return fmt.Errorf("specified address %x is outside the mapping range [%x, %x] for ObjectFile %q", addr, f.m.start, f.m.limit, path) + return fmt.Errorf("specified address %x is outside the mapping range [%x, %x] for ObjectFile %q", addr, f.m.start, f.m.limit, f.Path) } - ef, err := elfOpen(path) + ef, err := elfOpen(f.Path) if err != nil { - return fmt.Errorf("error parsing %s: %v", path, err) + return fmt.Errorf("error parsing %s: %v", f.Path, err) } defer ef.Close() ph, err := f.m.findProgramHeader(ef, addr) if err != nil { - return fmt.Errorf("failed to find program header for ObjectFile %q, ELF mapping %#v, address %x: %v", path, *f.m, addr, err) + return fmt.Errorf("failed to find program header for ObjectFile %q, ELF mapping %#v, address %x: %v", f.Path, *f.m, addr, err) } base, err := elfexec.GetBase(&ef.FileHeader, ph, f.m.kernelOffset, f.m.start, f.m.limit, f.m.offset) diff --git a/pkg/objectfile/object_file_test.go b/pkg/objectfile/object_file_test.go index 3361553420..8798ab5d0f 100644 --- a/pkg/objectfile/object_file_test.go +++ b/pkg/objectfile/object_file_test.go @@ -196,7 +196,6 @@ func TestELFObjAddr(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { o, err := open(name, tc.start, tc.limit, tc.offset, "") - o.path = name if (err != nil) != tc.wantOpenError { t.Errorf("openELF got error %v, want any error=%v", err, tc.wantOpenError) } diff --git a/pkg/profiler/profiler.go b/pkg/profiler/profiler.go index ee0d447e48..56f62b4eff 100644 --- a/pkg/profiler/profiler.go +++ b/pkg/profiler/profiler.go @@ -121,7 +121,7 @@ type CgroupProfiler struct { pidMappingFileCache *maps.PIDMappingFileCache perfCache *perf.Cache ksymCache *ksym.Cache - objCache *objectfile.Cache + objCache objectfile.Cache bpfMaps *bpfMaps @@ -140,7 +140,7 @@ func NewCgroupProfiler( logger log.Logger, reg prometheus.Registerer, ksymCache *ksym.Cache, - objCache *objectfile.Cache, + objCache objectfile.Cache, writeClient profilestorepb.ProfileStoreServiceClient, debugInfoClient debuginfo.Client, target model.LabelSet, @@ -447,7 +447,7 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time) level.Debug(p.logger).Log("msg", "failed to get mapping", "err", err) } - var objFile *objectfile.ObjectFile + var objFile *objectfile.MappedObjectFile if m != nil { objFile, err = p.objCache.ObjectFileForProcess(pid, m) if err != nil { @@ -462,8 +462,6 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time) if err != nil { level.Debug(p.logger).Log("msg", "failed to get normalized address from object file", "err", err) } else { - // TODO(kakkoyun): Remove! - level.Debug(p.logger).Log("msg", "transforming address", "pid", pid, "addr", addr, "normalizedAddr", normalizedAddr) normalizedAddr = nAddr } } @@ -510,12 +508,22 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, captureTime time.Time) prof.Sample = append(prof.Sample, s) } - var objFiles []*objectfile.ObjectFile - prof.Mapping, objFiles = mapping.AllMappings() + var mappedFiles []maps.ProcessMapping + prof.Mapping, mappedFiles = mapping.AllMappings() prof.Location = locations // Upload debug information of the discovered object files. - go p.debugInfoExtractor.EnsureUploaded(ctx, objFiles) + go func() { + var mObjFiles []*objectfile.MappedObjectFile + for _, mf := range mappedFiles { + mObjFile, err := p.objCache.ObjectFileForProcess(mf.PID, mf.Mapping) + if err != nil { + level.Debug(p.logger).Log("msg", "failed to open object file", "err", err) + } + mObjFiles = append(mObjFiles, mObjFile) + } + p.debugInfoExtractor.EnsureUploaded(ctx, mObjFiles) + }() // Resolve Kernel function names. kernelSymbols, err := p.ksymCache.Resolve(kernelAddresses) diff --git a/pkg/target/manager.go b/pkg/target/manager.go index 164b01e186..5078e0d02a 100644 --- a/pkg/target/manager.go +++ b/pkg/target/manager.go @@ -36,7 +36,6 @@ type Manager struct { reg prometheus.Registerer externalLabels model.LabelSet ksymCache *ksym.Cache - objCache *objectfile.Cache writeClient profilestorepb.ProfileStoreServiceClient debugInfoClient debuginfo.Client profilingDuration time.Duration @@ -46,8 +45,6 @@ type Manager struct { func NewManager( logger log.Logger, reg prometheus.Registerer, - ksymCache *ksym.Cache, - objCache *objectfile.Cache, writeClient profilestorepb.ProfileStoreServiceClient, debugInfoClient debuginfo.Client, profilingDuration time.Duration, @@ -60,8 +57,7 @@ func NewManager( logger: logger, reg: reg, externalLabels: externalLabels, - ksymCache: ksymCache, - objCache: objCache, + ksymCache: ksym.NewKsymCache(logger), writeClient: writeClient, debugInfoClient: debugInfoClient, profilingDuration: profilingDuration, @@ -88,11 +84,18 @@ func (m *Manager) reconcileTargets(ctx context.Context, targetSets map[string][] defer m.mtx.Unlock() level.Debug(m.logger).Log("msg", "reconciling targets") - for name, targetSet := range targetSets { pp, found := m.profilerPools[name] if !found { - pp = NewProfilerPool(ctx, m.logger, m.reg, m.ksymCache, m.objCache, m.writeClient, m.debugInfoClient, m.profilingDuration, m.externalLabels, m.tmp) + // An arbitrary coefficient. Number of assumed object files per target. + cacheSize := len(targetSet) * 5 + pp = NewProfilerPool( + ctx, m.logger, m.reg, + m.ksymCache, objectfile.NewCache(m.logger, cacheSize), + m.writeClient, m.debugInfoClient, + m.profilingDuration, m.externalLabels, + m.tmp, + ) m.profilerPools[name] = pp } diff --git a/pkg/target/profiler_pool.go b/pkg/target/profiler_pool.go index fa7b566aad..3a765c2d43 100644 --- a/pkg/target/profiler_pool.go +++ b/pkg/target/profiler_pool.go @@ -52,7 +52,7 @@ type ProfilerPool struct { logger log.Logger reg prometheus.Registerer ksymCache *ksym.Cache - objCache *objectfile.Cache + objCache objectfile.Cache writeClient profilestorepb.ProfileStoreServiceClient debugInfoClient debuginfo.Client profilingDuration time.Duration @@ -64,7 +64,7 @@ func NewProfilerPool( logger log.Logger, reg prometheus.Registerer, ksymCache *ksym.Cache, - objCache *objectfile.Cache, + objCache objectfile.Cache, writeClient profilestorepb.ProfileStoreServiceClient, debugInfoClient debuginfo.Client, profilingDuration time.Duration,