From a0d6231f867e05c4878bd739b2025746bccbecc1 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Tue, 19 Sep 2023 17:30:32 +0100 Subject: [PATCH 01/14] call underlying os.Hostname instad of the entire hostInfo gopsutil call --- src/core/environment.go | 10 ++++------ .../github.com/nginx/agent/v2/src/core/environment.go | 10 ++++------ .../github.com/nginx/agent/v2/src/core/environment.go | 10 ++++------ 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 4fdaaa401..e90fd3855 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -185,14 +185,12 @@ func getUnixName() string { } func (env *EnvironmentType) GetHostname() string { - ctx := context.Background() - defer ctx.Done() - hostInformation, err := host.InfoWithContext(ctx) + hostname, err := os.Hostname() if err != nil { - log.Warnf("Unable to read hostname from dataplane, defaulting value. Error: %v", err) - return "" + log.Warnf("unable to get hostname: %v", err) + hostname = "" } - return hostInformation.Hostname + return hostname } func (env *EnvironmentType) GetSystemUUID() string { diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 4fdaaa401..e90fd3855 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -185,14 +185,12 @@ func getUnixName() string { } func (env *EnvironmentType) GetHostname() string { - ctx := context.Background() - defer ctx.Done() - hostInformation, err := host.InfoWithContext(ctx) + hostname, err := os.Hostname() if err != nil { - log.Warnf("Unable to read hostname from dataplane, defaulting value. Error: %v", err) - return "" + log.Warnf("unable to get hostname: %v", err) + hostname = "" } - return hostInformation.Hostname + return hostname } func (env *EnvironmentType) GetSystemUUID() string { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 4fdaaa401..e90fd3855 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -185,14 +185,12 @@ func getUnixName() string { } func (env *EnvironmentType) GetHostname() string { - ctx := context.Background() - defer ctx.Done() - hostInformation, err := host.InfoWithContext(ctx) + hostname, err := os.Hostname() if err != nil { - log.Warnf("Unable to read hostname from dataplane, defaulting value. Error: %v", err) - return "" + log.Warnf("unable to get hostname: %v", err) + hostname = "" } - return hostInformation.Hostname + return hostname } func (env *EnvironmentType) GetSystemUUID() string { From becd61fcea7e18df04fa35f0fb8b7bd489ae78fd Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Tue, 19 Sep 2023 17:38:38 +0100 Subject: [PATCH 02/14] get hostID --- src/core/environment.go | 4 ++-- .../vendor/github.com/nginx/agent/v2/src/core/environment.go | 4 ++-- .../vendor/github.com/nginx/agent/v2/src/core/environment.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index e90fd3855..776a6bbe2 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -206,12 +206,12 @@ func (env *EnvironmentType) GetSystemUUID() string { return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() } - hostInfo, err := host.InfoWithContext(ctx) + hostID, err := host.HostIDWithContext(ctx) if err != nil { log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "" } - return uuid.NewMD5(uuid.Nil, []byte(hostInfo.HostID)).String() + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index e90fd3855..776a6bbe2 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -206,12 +206,12 @@ func (env *EnvironmentType) GetSystemUUID() string { return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() } - hostInfo, err := host.InfoWithContext(ctx) + hostID, err := host.HostIDWithContext(ctx) if err != nil { log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "" } - return uuid.NewMD5(uuid.Nil, []byte(hostInfo.HostID)).String() + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index e90fd3855..776a6bbe2 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -206,12 +206,12 @@ func (env *EnvironmentType) GetSystemUUID() string { return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() } - hostInfo, err := host.InfoWithContext(ctx) + hostID, err := host.HostIDWithContext(ctx) if err != nil { log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "" } - return uuid.NewMD5(uuid.Nil, []byte(hostInfo.HostID)).String() + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { From 466a89540b77ec224615c707d46fbc9c71c2b895 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Wed, 20 Sep 2023 16:01:07 +0100 Subject: [PATCH 03/14] wip --- src/core/environment.go | 28 ++- src/core/metrics/collectors/system.go | 2 +- src/core/metrics/sources/disk.go | 4 +- src/core/metrics/sources/disk_test.go | 9 +- src/plugins/dataplane_status.go | 2 +- .../nginx/agent/v2/src/core/environment.go | 28 ++- .../x/sync/singleflight/singleflight.go | 205 ++++++++++++++++++ test/integration/vendor/modules.txt | 1 + .../nginx/agent/v2/src/core/environment.go | 28 ++- .../v2/src/core/metrics/collectors/system.go | 2 +- .../agent/v2/src/core/metrics/sources/disk.go | 4 +- .../agent/v2/src/plugins/dataplane_status.go | 2 +- .../x/sync/singleflight/singleflight.go | 205 ++++++++++++++++++ test/performance/vendor/modules.txt | 1 + .../x/sync/singleflight/singleflight.go | 205 ++++++++++++++++++ vendor/modules.txt | 1 + 16 files changed, 682 insertions(+), 45 deletions(-) create mode 100644 test/integration/vendor/golang.org/x/sync/singleflight/singleflight.go create mode 100644 test/performance/vendor/golang.org/x/sync/singleflight/singleflight.go create mode 100644 vendor/golang.org/x/sync/singleflight/singleflight.go diff --git a/src/core/environment.go b/src/core/environment.go index 776a6bbe2..80a134b8a 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -30,6 +30,7 @@ import ( "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" "golang.org/x/sys/unix" "github.com/nginx/agent/sdk/v2/files" @@ -64,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,7 +96,7 @@ const ( var ( virtualizationFunc = host.VirtualizationWithContext - _ Environment = &EnvironmentType{} + singleflightGroup = &singleflight.Group{} basePattern = regexp.MustCompile("/([a-f0-9]{64})$") colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) @@ -307,18 +308,21 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { - if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) - return true + res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { + if _, err := os.Stat(filename); err == nil { + log.Debugf("is a container because (%s) exists", filename) + return true, nil + } } - } - // v1 check - if result, err := cGroupV1Check(selfCgroup); err == nil && result { - return result - } + // v1 check + if result, err := cGroupV1Check(selfCgroup); err == nil && result { + return result, err + } + return false, nil + }) - return false + return res.(bool) } // cGroupV1Check returns if running cgroup v1 diff --git a/src/core/metrics/collectors/system.go b/src/core/metrics/collectors/system.go index 0aa1060b3..a92c4a7b8 100644 --- a/src/core/metrics/collectors/system.go +++ b/src/core/metrics/collectors/system.go @@ -40,7 +40,7 @@ func NewSystemCollector(env core.Environment, conf *config.Config) *SystemCollec systemSources = []metrics.Source{ sources.NewVirtualMemorySource(sources.SystemNamespace, env), sources.NewCPUTimesSource(sources.SystemNamespace, env), - sources.NewDiskSource(sources.SystemNamespace), + sources.NewDiskSource(sources.SystemNamespace, env), sources.NewDiskIOSource(sources.SystemNamespace, env), sources.NewNetIOSource(sources.SystemNamespace, env), sources.NewLoadSource(sources.SystemNamespace), diff --git a/src/core/metrics/sources/disk.go b/src/core/metrics/sources/disk.go index 29b788953..fd888daf9 100644 --- a/src/core/metrics/sources/disk.go +++ b/src/core/metrics/sources/disk.go @@ -13,6 +13,7 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/shirou/gopsutil/v3/disk" ) @@ -23,9 +24,10 @@ type Disk struct { logger *MetricSourceLogger *namedMetric disks []disk.PartitionStat + } -func NewDiskSource(namespace string) *Disk { +func NewDiskSource(namespace string, env core.Environment) *Disk { ctx := context.Background() defer ctx.Done() disks, _ := disk.PartitionsWithContext(ctx, false) diff --git a/src/core/metrics/sources/disk_test.go b/src/core/metrics/sources/disk_test.go index 44a54bfc6..da05ecaf5 100644 --- a/src/core/metrics/sources/disk_test.go +++ b/src/core/metrics/sources/disk_test.go @@ -14,13 +14,15 @@ import ( "testing" "github.com/nginx/agent/v2/src/core/metrics" + tutils "github.com/nginx/agent/v2/test/utils" "github.com/stretchr/testify/assert" ) func TestNewDiskSource(t *testing.T) { namespace := "test" - actual := NewDiskSource(namespace) + env := tutils.GetMockEnv() + actual := NewDiskSource(namespace, env) assert.Equal(t, "disk", actual.group) assert.Equal(t, namespace, actual.namespace) @@ -29,8 +31,9 @@ func TestNewDiskSource(t *testing.T) { func TestDiskCollect(t *testing.T) { namespace := "test" - disk := NewDiskSource(namespace) - + env := tutils.GetMockEnv() + disk := NewDiskSource(namespace, env) + ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) diff --git a/src/plugins/dataplane_status.go b/src/plugins/dataplane_status.go index d1aef6ae7..d9f14f549 100644 --- a/src/plugins/dataplane_status.go +++ b/src/plugins/dataplane_status.go @@ -210,7 +210,7 @@ func (dps *DataPlaneStatus) dataplaneStatus(forceDetails bool) *proto.DataplaneS func (dps *DataPlaneStatus) hostInfo(send bool) (info *proto.HostInfo) { // this sets send if we are forcing details, or it has been 24 hours since the last send - hostInfo := dps.env.NewHostInfo(dps.version, dps.tags, dps.configDirs, true) + hostInfo := dps.env.NewHostInfo(dps.version, dps.tags, dps.configDirs, send) if !send && cmp.Equal(dps.envHostInfo, hostInfo) { return nil } diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 776a6bbe2..80a134b8a 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -30,6 +30,7 @@ import ( "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" "golang.org/x/sys/unix" "github.com/nginx/agent/sdk/v2/files" @@ -64,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,7 +96,7 @@ const ( var ( virtualizationFunc = host.VirtualizationWithContext - _ Environment = &EnvironmentType{} + singleflightGroup = &singleflight.Group{} basePattern = regexp.MustCompile("/([a-f0-9]{64})$") colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) @@ -307,18 +308,21 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { - if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) - return true + res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { + if _, err := os.Stat(filename); err == nil { + log.Debugf("is a container because (%s) exists", filename) + return true, nil + } } - } - // v1 check - if result, err := cGroupV1Check(selfCgroup); err == nil && result { - return result - } + // v1 check + if result, err := cGroupV1Check(selfCgroup); err == nil && result { + return result, err + } + return false, nil + }) - return false + return res.(bool) } // cGroupV1Check returns if running cgroup v1 diff --git a/test/integration/vendor/golang.org/x/sync/singleflight/singleflight.go b/test/integration/vendor/golang.org/x/sync/singleflight/singleflight.go new file mode 100644 index 000000000..8473fb792 --- /dev/null +++ b/test/integration/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -0,0 +1,205 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight // import "golang.org/x/sync/singleflight" + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val interface{} + err error + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result struct { + Val interface{} + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { + ch := make(chan Result, 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call{chans: []chan<- Result{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + g.mu.Lock() + defer g.mu.Unlock() + c.wg.Done() + if g.m[key] == c { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group) Forget(key string) { + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() +} diff --git a/test/integration/vendor/modules.txt b/test/integration/vendor/modules.txt index 90eb66298..4d683e6a4 100644 --- a/test/integration/vendor/modules.txt +++ b/test/integration/vendor/modules.txt @@ -978,6 +978,7 @@ golang.org/x/oauth2/internal ## explicit; go 1.17 golang.org/x/sync/errgroup golang.org/x/sync/semaphore +golang.org/x/sync/singleflight # golang.org/x/sys v0.11.0 ## explicit; go 1.17 golang.org/x/sys/cpu diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 776a6bbe2..80a134b8a 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -30,6 +30,7 @@ import ( "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" "golang.org/x/sys/unix" "github.com/nginx/agent/sdk/v2/files" @@ -64,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,7 +96,7 @@ const ( var ( virtualizationFunc = host.VirtualizationWithContext - _ Environment = &EnvironmentType{} + singleflightGroup = &singleflight.Group{} basePattern = regexp.MustCompile("/([a-f0-9]{64})$") colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) @@ -307,18 +308,21 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { - if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) - return true + res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { + if _, err := os.Stat(filename); err == nil { + log.Debugf("is a container because (%s) exists", filename) + return true, nil + } } - } - // v1 check - if result, err := cGroupV1Check(selfCgroup); err == nil && result { - return result - } + // v1 check + if result, err := cGroupV1Check(selfCgroup); err == nil && result { + return result, err + } + return false, nil + }) - return false + return res.(bool) } // cGroupV1Check returns if running cgroup v1 diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go index 0aa1060b3..a92c4a7b8 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go @@ -40,7 +40,7 @@ func NewSystemCollector(env core.Environment, conf *config.Config) *SystemCollec systemSources = []metrics.Source{ sources.NewVirtualMemorySource(sources.SystemNamespace, env), sources.NewCPUTimesSource(sources.SystemNamespace, env), - sources.NewDiskSource(sources.SystemNamespace), + sources.NewDiskSource(sources.SystemNamespace, env), sources.NewDiskIOSource(sources.SystemNamespace, env), sources.NewNetIOSource(sources.SystemNamespace, env), sources.NewLoadSource(sources.SystemNamespace), diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go index 29b788953..fd888daf9 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go @@ -13,6 +13,7 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/shirou/gopsutil/v3/disk" ) @@ -23,9 +24,10 @@ type Disk struct { logger *MetricSourceLogger *namedMetric disks []disk.PartitionStat + } -func NewDiskSource(namespace string) *Disk { +func NewDiskSource(namespace string, env core.Environment) *Disk { ctx := context.Background() defer ctx.Done() disks, _ := disk.PartitionsWithContext(ctx, false) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/dataplane_status.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/dataplane_status.go index d1aef6ae7..d9f14f549 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/dataplane_status.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/dataplane_status.go @@ -210,7 +210,7 @@ func (dps *DataPlaneStatus) dataplaneStatus(forceDetails bool) *proto.DataplaneS func (dps *DataPlaneStatus) hostInfo(send bool) (info *proto.HostInfo) { // this sets send if we are forcing details, or it has been 24 hours since the last send - hostInfo := dps.env.NewHostInfo(dps.version, dps.tags, dps.configDirs, true) + hostInfo := dps.env.NewHostInfo(dps.version, dps.tags, dps.configDirs, send) if !send && cmp.Equal(dps.envHostInfo, hostInfo) { return nil } diff --git a/test/performance/vendor/golang.org/x/sync/singleflight/singleflight.go b/test/performance/vendor/golang.org/x/sync/singleflight/singleflight.go new file mode 100644 index 000000000..8473fb792 --- /dev/null +++ b/test/performance/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -0,0 +1,205 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight // import "golang.org/x/sync/singleflight" + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val interface{} + err error + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result struct { + Val interface{} + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { + ch := make(chan Result, 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call{chans: []chan<- Result{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + g.mu.Lock() + defer g.mu.Unlock() + c.wg.Done() + if g.m[key] == c { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group) Forget(key string) { + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() +} diff --git a/test/performance/vendor/modules.txt b/test/performance/vendor/modules.txt index db5d0da4a..f13622221 100644 --- a/test/performance/vendor/modules.txt +++ b/test/performance/vendor/modules.txt @@ -338,6 +338,7 @@ golang.org/x/net/trace # golang.org/x/sync v0.3.0 ## explicit; go 1.17 golang.org/x/sync/errgroup +golang.org/x/sync/singleflight # golang.org/x/sys v0.11.0 ## explicit; go 1.17 golang.org/x/sys/cpu diff --git a/vendor/golang.org/x/sync/singleflight/singleflight.go b/vendor/golang.org/x/sync/singleflight/singleflight.go new file mode 100644 index 000000000..8473fb792 --- /dev/null +++ b/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -0,0 +1,205 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight // import "golang.org/x/sync/singleflight" + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val interface{} + err error + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result struct { + Val interface{} + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { + ch := make(chan Result, 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call{chans: []chan<- Result{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + g.mu.Lock() + defer g.mu.Unlock() + c.wg.Done() + if g.m[key] == c { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group) Forget(key string) { + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f73f4696f..02d4349e8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1635,6 +1635,7 @@ golang.org/x/net/trace ## explicit; go 1.17 golang.org/x/sync/errgroup golang.org/x/sync/semaphore +golang.org/x/sync/singleflight # golang.org/x/sys v0.11.0 ## explicit; go 1.17 golang.org/x/sys/cpu From 3bcc45c810f77b1663403e4a64a22cabe54533b4 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Thu, 21 Sep 2023 13:53:07 +0100 Subject: [PATCH 04/14] local changes --- src/core/environment.go | 16 ++++++++-------- src/core/metrics/sources/disk.go | 1 - src/core/metrics/sources/disk_test.go | 2 +- .../nginx/agent/v2/src/core/environment.go | 16 ++++++++-------- .../nginx/agent/v2/src/core/environment.go | 16 ++++++++-------- .../agent/v2/src/core/metrics/sources/disk.go | 1 - 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 80a134b8a..8706ff93b 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -65,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,13 +95,13 @@ const ( ) var ( - virtualizationFunc = host.VirtualizationWithContext - singleflightGroup = &singleflight.Group{} - basePattern = regexp.MustCompile("/([a-f0-9]{64})$") - colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") - scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) - containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") - containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") + virtualizationFunc = host.VirtualizationWithContext + singleflightGroup = &singleflight.Group{} + basePattern = regexp.MustCompile("/([a-f0-9]{64})$") + colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") + scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) + containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") + containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") ) func (env *EnvironmentType) NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo { diff --git a/src/core/metrics/sources/disk.go b/src/core/metrics/sources/disk.go index fd888daf9..6daa58e7a 100644 --- a/src/core/metrics/sources/disk.go +++ b/src/core/metrics/sources/disk.go @@ -24,7 +24,6 @@ type Disk struct { logger *MetricSourceLogger *namedMetric disks []disk.PartitionStat - } func NewDiskSource(namespace string, env core.Environment) *Disk { diff --git a/src/core/metrics/sources/disk_test.go b/src/core/metrics/sources/disk_test.go index da05ecaf5..65e4e3bc0 100644 --- a/src/core/metrics/sources/disk_test.go +++ b/src/core/metrics/sources/disk_test.go @@ -33,7 +33,7 @@ func TestDiskCollect(t *testing.T) { namespace := "test" env := tutils.GetMockEnv() disk := NewDiskSource(namespace, env) - + ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 80a134b8a..8706ff93b 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -65,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,13 +95,13 @@ const ( ) var ( - virtualizationFunc = host.VirtualizationWithContext - singleflightGroup = &singleflight.Group{} - basePattern = regexp.MustCompile("/([a-f0-9]{64})$") - colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") - scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) - containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") - containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") + virtualizationFunc = host.VirtualizationWithContext + singleflightGroup = &singleflight.Group{} + basePattern = regexp.MustCompile("/([a-f0-9]{64})$") + colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") + scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) + containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") + containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") ) func (env *EnvironmentType) NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 80a134b8a..8706ff93b 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -65,7 +65,7 @@ type ConfigApplyMarker interface { } type EnvironmentType struct { - host *proto.HostInfo + host *proto.HostInfo } type Process struct { @@ -95,13 +95,13 @@ const ( ) var ( - virtualizationFunc = host.VirtualizationWithContext - singleflightGroup = &singleflight.Group{} - basePattern = regexp.MustCompile("/([a-f0-9]{64})$") - colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") - scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) - containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") - containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") + virtualizationFunc = host.VirtualizationWithContext + singleflightGroup = &singleflight.Group{} + basePattern = regexp.MustCompile("/([a-f0-9]{64})$") + colonPattern = regexp.MustCompile(":([a-f0-9]{64})$") + scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`) + containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})") + containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})") ) func (env *EnvironmentType) NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go index fd888daf9..6daa58e7a 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go @@ -24,7 +24,6 @@ type Disk struct { logger *MetricSourceLogger *namedMetric disks []disk.PartitionStat - } func NewDiskSource(namespace string, env core.Environment) *Disk { From 6f55e7af031c07987753118f3ad7476721df1e01 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Thu, 21 Sep 2023 15:15:59 +0100 Subject: [PATCH 05/14] modified the calls to use singleflight --- src/core/environment.go | 64 +++++++++++-------- .../nginx/agent/v2/src/core/environment.go | 62 ++++++++++-------- .../nginx/agent/v2/src/core/environment.go | 62 ++++++++++-------- 3 files changed, 109 insertions(+), 79 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 8706ff93b..4f8b22c66 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -92,6 +92,9 @@ const ( KernProc = 14 // struct: process entries KernProcPathname = 12 // path to executable SYS_SYSCTL = 202 + IsContainerKey = "isContainer" + GetContainerIDKey = "GetContainerID" + GetSystemUUIDKey = "GetSystemUUIDKey" ) var ( @@ -194,25 +197,28 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { - ctx := context.Background() - defer ctx.Done() +func (env *EnvironmentType) GetSystemUUID() string { + res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + var err error + ctx := context.Background() + defer ctx.Done() - if env.IsContainer() { - containerID, err := env.GetContainerID() - if err != nil { - log.Errorf("Unable to read docker container ID: %v", err) - return "" + if env.IsContainer() { + containerID, err := env.GetContainerID() + if err != nil { + return "", errors.New(fmt.Sprintf("Unable to read docker container ID: %v", err)) + } + return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), err } - return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() - } - hostID, err := host.HostIDWithContext(ctx) - if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) - return "" - } - return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() + hostID, err := host.HostIDWithContext(ctx) + if err != nil { + log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) + return "", err + } + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err + }) + return res.(string) } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { @@ -308,7 +314,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -349,20 +355,24 @@ func cGroupV1Check(cgroupFile string) (bool, error) { // GetContainerID returns the ID of the current environment if running in a container func (env *EnvironmentType) GetContainerID() (string, error) { - const mountInfo = "/proc/self/mountinfo" + res, err, _ := singleflightGroup.Do(GetContainerIDKey, func() (interface{}, error) { + const mountInfo = "/proc/self/mountinfo" - if !env.IsContainer() { - return "", errors.New("not in docker") - } + if !env.IsContainer() { + return "", errors.New("not in docker") + } - containerID, err := getContainerID(mountInfo) - if err != nil { - return "", fmt.Errorf("could not get container ID: %v", err) - } + containerID, err := getContainerID(mountInfo) + if err != nil { + return "", fmt.Errorf("could not get container ID: %v", err) + } - log.Debugf("Container ID: %s", containerID) + log.Debugf("Container ID: %s", containerID) + + return containerID, err + }) - return containerID, err + return res.(string), err } // getContainerID returns the container ID of the current running environment. diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 8706ff93b..e5a3d0547 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -92,6 +92,9 @@ const ( KernProc = 14 // struct: process entries KernProcPathname = 12 // path to executable SYS_SYSCTL = 202 + IsContainerKey = "isContainer" + GetContainerIDKey = "GetContainerID" + GetSystemUUIDKey = "GetSystemUUIDKey" ) var ( @@ -194,25 +197,28 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { - ctx := context.Background() - defer ctx.Done() +func (env *EnvironmentType) GetSystemUUID() string { + res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + ctx := context.Background() + defer ctx.Done() + + if env.IsContainer() { + containerID, err := env.GetContainerID() + if err != nil { + log.Errorf("Unable to read docker container ID: %v", err) + return "" + } + return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), nil + } - if env.IsContainer() { - containerID, err := env.GetContainerID() + hostID, err := host.HostIDWithContext(ctx) if err != nil { - log.Errorf("Unable to read docker container ID: %v", err) + log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "" } - return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() - } - - hostID, err := host.HostIDWithContext(ctx) - if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) - return "" - } - return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), nil + }) + return res.(string) } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { @@ -308,7 +314,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -349,20 +355,24 @@ func cGroupV1Check(cgroupFile string) (bool, error) { // GetContainerID returns the ID of the current environment if running in a container func (env *EnvironmentType) GetContainerID() (string, error) { - const mountInfo = "/proc/self/mountinfo" + res, err, _ := singleflightGroup.Do(GetContainerIDKey, func() (interface{}, error) { + const mountInfo = "/proc/self/mountinfo" - if !env.IsContainer() { - return "", errors.New("not in docker") - } + if !env.IsContainer() { + return "", errors.New("not in docker") + } - containerID, err := getContainerID(mountInfo) - if err != nil { - return "", fmt.Errorf("could not get container ID: %v", err) - } + containerID, err := getContainerID(mountInfo) + if err != nil { + return "", fmt.Errorf("could not get container ID: %v", err) + } - log.Debugf("Container ID: %s", containerID) + log.Debugf("Container ID: %s", containerID) + + return containerID, err + }) - return containerID, err + return res.(string), err } // getContainerID returns the container ID of the current running environment. diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 8706ff93b..e5a3d0547 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -92,6 +92,9 @@ const ( KernProc = 14 // struct: process entries KernProcPathname = 12 // path to executable SYS_SYSCTL = 202 + IsContainerKey = "isContainer" + GetContainerIDKey = "GetContainerID" + GetSystemUUIDKey = "GetSystemUUIDKey" ) var ( @@ -194,25 +197,28 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { - ctx := context.Background() - defer ctx.Done() +func (env *EnvironmentType) GetSystemUUID() string { + res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + ctx := context.Background() + defer ctx.Done() + + if env.IsContainer() { + containerID, err := env.GetContainerID() + if err != nil { + log.Errorf("Unable to read docker container ID: %v", err) + return "" + } + return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), nil + } - if env.IsContainer() { - containerID, err := env.GetContainerID() + hostID, err := host.HostIDWithContext(ctx) if err != nil { - log.Errorf("Unable to read docker container ID: %v", err) + log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "" } - return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String() - } - - hostID, err := host.HostIDWithContext(ctx) - if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) - return "" - } - return uuid.NewMD5(uuid.Nil, []byte(hostID)).String() + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), nil + }) + return res.(string) } func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) { @@ -308,7 +314,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do("isContainer", func() (interface{}, error) { + res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -349,20 +355,24 @@ func cGroupV1Check(cgroupFile string) (bool, error) { // GetContainerID returns the ID of the current environment if running in a container func (env *EnvironmentType) GetContainerID() (string, error) { - const mountInfo = "/proc/self/mountinfo" + res, err, _ := singleflightGroup.Do(GetContainerIDKey, func() (interface{}, error) { + const mountInfo = "/proc/self/mountinfo" - if !env.IsContainer() { - return "", errors.New("not in docker") - } + if !env.IsContainer() { + return "", errors.New("not in docker") + } - containerID, err := getContainerID(mountInfo) - if err != nil { - return "", fmt.Errorf("could not get container ID: %v", err) - } + containerID, err := getContainerID(mountInfo) + if err != nil { + return "", fmt.Errorf("could not get container ID: %v", err) + } - log.Debugf("Container ID: %s", containerID) + log.Debugf("Container ID: %s", containerID) + + return containerID, err + }) - return containerID, err + return res.(string), err } // getContainerID returns the container ID of the current running environment. From 4afe84d038a9d93f883073b3437f10bd1dcef397 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Thu, 21 Sep 2023 15:16:59 +0100 Subject: [PATCH 06/14] modified the calls to use singleflight --- src/core/environment.go | 2 +- .../nginx/agent/v2/src/core/environment.go | 12 ++++++------ .../nginx/agent/v2/src/core/environment.go | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 4f8b22c66..f9d9de862 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -197,7 +197,7 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { +func (env *EnvironmentType) GetSystemUUID() string { res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { var err error ctx := context.Background() diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index e5a3d0547..f9d9de862 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -197,26 +197,26 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { +func (env *EnvironmentType) GetSystemUUID() string { res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + var err error ctx := context.Background() defer ctx.Done() if env.IsContainer() { containerID, err := env.GetContainerID() if err != nil { - log.Errorf("Unable to read docker container ID: %v", err) - return "" + return "", errors.New(fmt.Sprintf("Unable to read docker container ID: %v", err)) } - return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), nil + return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), err } hostID, err := host.HostIDWithContext(ctx) if err != nil { log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) - return "" + return "", err } - return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), nil + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) return res.(string) } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index e5a3d0547..f9d9de862 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -197,26 +197,26 @@ func (env *EnvironmentType) GetHostname() string { return hostname } -func (env *EnvironmentType) GetSystemUUID() string { +func (env *EnvironmentType) GetSystemUUID() string { res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + var err error ctx := context.Background() defer ctx.Done() if env.IsContainer() { containerID, err := env.GetContainerID() if err != nil { - log.Errorf("Unable to read docker container ID: %v", err) - return "" + return "", errors.New(fmt.Sprintf("Unable to read docker container ID: %v", err)) } - return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), nil + return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), err } hostID, err := host.HostIDWithContext(ctx) if err != nil { log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) - return "" + return "", err } - return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), nil + return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) return res.(string) } From 334cb4c1353c591f010a6699c8d98ec03145c07a Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Fri, 22 Sep 2023 14:32:02 +0100 Subject: [PATCH 07/14] change the disk call to environment.go --- src/core/environment.go | 8 +++ src/core/fake_environment_test.go | 71 +++++++++++++++++++ src/core/metrics/sources/disk.go | 4 +- .../nginx/agent/v2/src/core/environment.go | 8 +++ .../nginx/agent/v2/test/utils/environment.go | 13 ++++ .../nginx/agent/v2/src/core/environment.go | 8 +++ .../agent/v2/src/core/metrics/sources/disk.go | 4 +- .../nginx/agent/v2/test/utils/environment.go | 13 ++++ test/utils/environment.go | 13 ++++ 9 files changed, 136 insertions(+), 6 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index f9d9de862..8a2afa9ae 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -53,6 +53,7 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) + Disks() ([]disk.PartitionStat, error) DiskDevices() ([]string, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) @@ -442,6 +443,13 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } +func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { + ctx := context.Background() + defer ctx.Done() + disks, err := disk.PartitionsWithContext(ctx, false) + return disks, err +} + func (env *EnvironmentType) GetNetOverflow() (float64, error) { return network.GetNetOverflow() } diff --git a/src/core/fake_environment_test.go b/src/core/fake_environment_test.go index 02b4da669..8201ade22 100644 --- a/src/core/fake_environment_test.go +++ b/src/core/fake_environment_test.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" + "github.com/shirou/gopsutil/v3/disk" ) type FakeEnvironment struct { @@ -33,6 +34,18 @@ type FakeEnvironment struct { result1 []string result2 error } + DisksStub func() ([]disk.PartitionStat, error) + disksMutex sync.RWMutex + disksArgsForCall []struct { + } + disksReturns struct { + result1 []disk.PartitionStat + result2 error + } + disksReturnsOnCall map[int]struct { + result1 []disk.PartitionStat + result2 error + } FileStatStub func(string) (fs.FileInfo, error) fileStatMutex sync.RWMutex fileStatArgsForCall []struct { @@ -287,6 +300,62 @@ func (fake *FakeEnvironment) DiskDevicesReturnsOnCall(i int, result1 []string, r }{result1, result2} } +func (fake *FakeEnvironment) Disks() ([]disk.PartitionStat, error) { + fake.disksMutex.Lock() + ret, specificReturn := fake.disksReturnsOnCall[len(fake.disksArgsForCall)] + fake.disksArgsForCall = append(fake.disksArgsForCall, struct { + }{}) + stub := fake.DisksStub + fakeReturns := fake.disksReturns + fake.recordInvocation("Disks", []interface{}{}) + fake.disksMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeEnvironment) DisksCallCount() int { + fake.disksMutex.RLock() + defer fake.disksMutex.RUnlock() + return len(fake.disksArgsForCall) +} + +func (fake *FakeEnvironment) DisksCalls(stub func() ([]disk.PartitionStat, error)) { + fake.disksMutex.Lock() + defer fake.disksMutex.Unlock() + fake.DisksStub = stub +} + +func (fake *FakeEnvironment) DisksReturns(result1 []disk.PartitionStat, result2 error) { + fake.disksMutex.Lock() + defer fake.disksMutex.Unlock() + fake.DisksStub = nil + fake.disksReturns = struct { + result1 []disk.PartitionStat + result2 error + }{result1, result2} +} + +func (fake *FakeEnvironment) DisksReturnsOnCall(i int, result1 []disk.PartitionStat, result2 error) { + fake.disksMutex.Lock() + defer fake.disksMutex.Unlock() + fake.DisksStub = nil + if fake.disksReturnsOnCall == nil { + fake.disksReturnsOnCall = make(map[int]struct { + result1 []disk.PartitionStat + result2 error + }) + } + fake.disksReturnsOnCall[i] = struct { + result1 []disk.PartitionStat + result2 error + }{result1, result2} +} + func (fake *FakeEnvironment) FileStat(arg1 string) (fs.FileInfo, error) { fake.fileStatMutex.Lock() ret, specificReturn := fake.fileStatReturnsOnCall[len(fake.fileStatArgsForCall)] @@ -943,6 +1012,8 @@ func (fake *FakeEnvironment) Invocations() map[string][][]interface{} { defer fake.deleteFileMutex.RUnlock() fake.diskDevicesMutex.RLock() defer fake.diskDevicesMutex.RUnlock() + fake.disksMutex.RLock() + defer fake.disksMutex.RUnlock() fake.fileStatMutex.RLock() defer fake.fileStatMutex.RUnlock() fake.getContainerIDMutex.RLock() diff --git a/src/core/metrics/sources/disk.go b/src/core/metrics/sources/disk.go index 6daa58e7a..5b72ddb95 100644 --- a/src/core/metrics/sources/disk.go +++ b/src/core/metrics/sources/disk.go @@ -27,9 +27,7 @@ type Disk struct { } func NewDiskSource(namespace string, env core.Environment) *Disk { - ctx := context.Background() - defer ctx.Done() - disks, _ := disk.PartitionsWithContext(ctx, false) + disks, _ := env.Disks() return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} } diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index f9d9de862..8a2afa9ae 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -53,6 +53,7 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) + Disks() ([]disk.PartitionStat, error) DiskDevices() ([]string, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) @@ -442,6 +443,13 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } +func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { + ctx := context.Background() + defer ctx.Done() + disks, err := disk.PartitionsWithContext(ctx, false) + return disks, err +} + func (env *EnvironmentType) GetNetOverflow() (float64, error) { return network.GetNetOverflow() } diff --git a/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go index da6787a24..845b28124 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go @@ -5,6 +5,7 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" + "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -12,6 +13,18 @@ type MockEnvironment struct { mock.Mock } +// Disks implements core.Environment. +func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { + return []disk.PartitionStat{ + { + Device: "sd01", + Mountpoint: "/", + Fstype: "ext4", + Opts: []string{"ro"}, + }, + }, nil +} + func GetProcesses() []*core.Process { return []*core.Process{ {Pid: 1, Name: "12345", IsMaster: true}, diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index f9d9de862..8a2afa9ae 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -53,6 +53,7 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) + Disks() ([]disk.PartitionStat, error) DiskDevices() ([]string, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) @@ -442,6 +443,13 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } +func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { + ctx := context.Background() + defer ctx.Done() + disks, err := disk.PartitionsWithContext(ctx, false) + return disks, err +} + func (env *EnvironmentType) GetNetOverflow() (float64, error) { return network.GetNetOverflow() } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go index 6daa58e7a..5b72ddb95 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go @@ -27,9 +27,7 @@ type Disk struct { } func NewDiskSource(namespace string, env core.Environment) *Disk { - ctx := context.Background() - defer ctx.Done() - disks, _ := disk.PartitionsWithContext(ctx, false) + disks, _ := env.Disks() return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go index da6787a24..845b28124 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go @@ -5,6 +5,7 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" + "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -12,6 +13,18 @@ type MockEnvironment struct { mock.Mock } +// Disks implements core.Environment. +func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { + return []disk.PartitionStat{ + { + Device: "sd01", + Mountpoint: "/", + Fstype: "ext4", + Opts: []string{"ro"}, + }, + }, nil +} + func GetProcesses() []*core.Process { return []*core.Process{ {Pid: 1, Name: "12345", IsMaster: true}, diff --git a/test/utils/environment.go b/test/utils/environment.go index da6787a24..845b28124 100644 --- a/test/utils/environment.go +++ b/test/utils/environment.go @@ -5,6 +5,7 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" + "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -12,6 +13,18 @@ type MockEnvironment struct { mock.Mock } +// Disks implements core.Environment. +func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { + return []disk.PartitionStat{ + { + Device: "sd01", + Mountpoint: "/", + Fstype: "ext4", + Opts: []string{"ro"}, + }, + }, nil +} + func GetProcesses() []*core.Process { return []*core.Process{ {Pid: 1, Name: "12345", IsMaster: true}, From 84d5aae3802fcdd029da610555cd75ef5eb62541 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Fri, 22 Sep 2023 14:54:55 +0100 Subject: [PATCH 08/14] local changes --- sdk/config_helpers_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/config_helpers_test.go b/sdk/config_helpers_test.go index 4cae45d43..086a49243 100644 --- a/sdk/config_helpers_test.go +++ b/sdk/config_helpers_test.go @@ -2104,7 +2104,7 @@ func TestSslDirectives(t *testing.T) { }` // preparing test cases as well as expected results - var tests = []struct { + tests := []struct { algoName string config string expected struct { @@ -2251,6 +2251,5 @@ func TestSslDirectives(t *testing.T) { assert.Equal(t, certMeta.PublicKeyAlgorithm, cert.PublicKeyAlgorithm) } }) - } } From c048dba51d8aed86447f684ef91d9646a840d25c Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Mon, 25 Sep 2023 10:43:30 +0100 Subject: [PATCH 09/14] fix tests --- src/core/environment.go | 79 +++++++++----- src/core/fake_environment_test.go | 100 ++++++++++++++++-- src/core/metrics/sources/disk.go | 16 +-- src/core/metrics/sources/disk_test.go | 2 +- .../nginx/agent/v2/src/core/environment.go | 74 ++++++++----- .../nginx/agent/v2/test/utils/environment.go | 24 +++-- .../nginx/agent/v2/src/core/environment.go | 74 ++++++++----- .../agent/v2/src/core/metrics/sources/disk.go | 16 +-- .../nginx/agent/v2/test/utils/environment.go | 24 +++-- test/utils/environment.go | 24 +++-- 10 files changed, 309 insertions(+), 124 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index fe568901a..080c17f31 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -53,8 +53,9 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) - Disks() ([]disk.PartitionStat, error) + Disks() ([]*proto.DiskPartition, error) DiskDevices() ([]string, error) + DiskUsage(mountPoint string) (*DiskUsage, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) IsContainer() bool @@ -82,6 +83,13 @@ type Process struct { Command string } +type DiskUsage struct { + Total float64 + Used float64 + Free float64 + UsedPercentage float64 +} + const ( lengthOfContainerId = 64 versionId = "VERSION_ID" @@ -128,6 +136,11 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer tags = &[]string{} } + disks, err := env.Disks() + if err != nil { + log.Warnf("Unable to get disks information from the host: %v", err) + disks = nil + } hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -136,7 +149,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer OsType: hostInformation.OS, Uuid: env.GetSystemUUID(), Uname: getUnixName(), - Partitons: diskPartitions(), + Partitons: disks, Network: env.networks(), Processor: processors(hostInformation.KernelArch), Release: releaseInfo("/etc/os-release"), @@ -199,7 +212,7 @@ func (env *EnvironmentType) GetHostname() string { } func (env *EnvironmentType) GetSystemUUID() string { - res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { var err error ctx := context.Background() defer ctx.Done() @@ -219,6 +232,11 @@ func (env *EnvironmentType) GetSystemUUID() string { } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) + if err != nil { + log.Warnf("unable to set hostname due to %v", err) + return "" + } + return res.(string) } @@ -443,11 +461,40 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } -func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { +func (env *EnvironmentType) Disks() (partitions []*proto.DiskPartition, err error) { ctx := context.Background() defer ctx.Done() - disks, err := disk.PartitionsWithContext(ctx, false) - return disks, err + parts, err := disk.PartitionsWithContext(ctx, false) + if err != nil { + // return an array of 0 + log.Errorf("Could not read disk partitions for host: %v", err) + return []*proto.DiskPartition{}, err + } + for _, part := range parts { + pm := proto.DiskPartition{ + MountPoint: part.Mountpoint, + Device: part.Device, + FsType: part.Fstype, + } + partitions = append(partitions, &pm) + } + return partitions, nil +} + +func (env *EnvironmentType) DiskUsage(mountPoint string) (*DiskUsage, error) { + ctx := context.Background() + defer ctx.Done() + usage, err := disk.UsageWithContext(ctx, mountPoint) + if err != nil { + return nil, errors.New("unable to obtain disk usage stats") + } + + return &DiskUsage{ + Total: float64(usage.Total), + Used: float64(usage.Used), + Free: float64(usage.Free), + UsedPercentage: float64(usage.UsedPercent), + }, nil } func (env *EnvironmentType) GetNetOverflow() (float64, error) { @@ -817,26 +864,6 @@ func virtualization() (string, string) { return virtualizationSystem, virtualizationRole } -func diskPartitions() (partitions []*proto.DiskPartition) { - ctx := context.Background() - defer ctx.Done() - parts, err := disk.PartitionsWithContext(ctx, false) - if err != nil { - // return an array of 0 - log.Errorf("Could not read disk partitions for host: %v", err) - return []*proto.DiskPartition{} - } - for _, part := range parts { - pm := proto.DiskPartition{ - MountPoint: part.Mountpoint, - Device: part.Device, - FsType: part.Fstype, - } - partitions = append(partitions, &pm) - } - return partitions -} - func releaseInfo(osReleaseFile string) (release *proto.ReleaseInfo) { hostReleaseInfo := getHostReleaseInfo() osRelease, err := getOsRelease(osReleaseFile) diff --git a/src/core/fake_environment_test.go b/src/core/fake_environment_test.go index 8201ade22..4de09b7a2 100644 --- a/src/core/fake_environment_test.go +++ b/src/core/fake_environment_test.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" - "github.com/shirou/gopsutil/v3/disk" ) type FakeEnvironment struct { @@ -34,16 +33,29 @@ type FakeEnvironment struct { result1 []string result2 error } - DisksStub func() ([]disk.PartitionStat, error) + DiskUsageStub func(string) (*DiskUsage, error) + diskUsageMutex sync.RWMutex + diskUsageArgsForCall []struct { + arg1 string + } + diskUsageReturns struct { + result1 *DiskUsage + result2 error + } + diskUsageReturnsOnCall map[int]struct { + result1 *DiskUsage + result2 error + } + DisksStub func() ([]*proto.DiskPartition, error) disksMutex sync.RWMutex disksArgsForCall []struct { } disksReturns struct { - result1 []disk.PartitionStat + result1 []*proto.DiskPartition result2 error } disksReturnsOnCall map[int]struct { - result1 []disk.PartitionStat + result1 []*proto.DiskPartition result2 error } FileStatStub func(string) (fs.FileInfo, error) @@ -300,7 +312,71 @@ func (fake *FakeEnvironment) DiskDevicesReturnsOnCall(i int, result1 []string, r }{result1, result2} } -func (fake *FakeEnvironment) Disks() ([]disk.PartitionStat, error) { +func (fake *FakeEnvironment) DiskUsage(arg1 string) (*DiskUsage, error) { + fake.diskUsageMutex.Lock() + ret, specificReturn := fake.diskUsageReturnsOnCall[len(fake.diskUsageArgsForCall)] + fake.diskUsageArgsForCall = append(fake.diskUsageArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DiskUsageStub + fakeReturns := fake.diskUsageReturns + fake.recordInvocation("DiskUsage", []interface{}{arg1}) + fake.diskUsageMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeEnvironment) DiskUsageCallCount() int { + fake.diskUsageMutex.RLock() + defer fake.diskUsageMutex.RUnlock() + return len(fake.diskUsageArgsForCall) +} + +func (fake *FakeEnvironment) DiskUsageCalls(stub func(string) (*DiskUsage, error)) { + fake.diskUsageMutex.Lock() + defer fake.diskUsageMutex.Unlock() + fake.DiskUsageStub = stub +} + +func (fake *FakeEnvironment) DiskUsageArgsForCall(i int) string { + fake.diskUsageMutex.RLock() + defer fake.diskUsageMutex.RUnlock() + argsForCall := fake.diskUsageArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeEnvironment) DiskUsageReturns(result1 *DiskUsage, result2 error) { + fake.diskUsageMutex.Lock() + defer fake.diskUsageMutex.Unlock() + fake.DiskUsageStub = nil + fake.diskUsageReturns = struct { + result1 *DiskUsage + result2 error + }{result1, result2} +} + +func (fake *FakeEnvironment) DiskUsageReturnsOnCall(i int, result1 *DiskUsage, result2 error) { + fake.diskUsageMutex.Lock() + defer fake.diskUsageMutex.Unlock() + fake.DiskUsageStub = nil + if fake.diskUsageReturnsOnCall == nil { + fake.diskUsageReturnsOnCall = make(map[int]struct { + result1 *DiskUsage + result2 error + }) + } + fake.diskUsageReturnsOnCall[i] = struct { + result1 *DiskUsage + result2 error + }{result1, result2} +} + +func (fake *FakeEnvironment) Disks() ([]*proto.DiskPartition, error) { fake.disksMutex.Lock() ret, specificReturn := fake.disksReturnsOnCall[len(fake.disksArgsForCall)] fake.disksArgsForCall = append(fake.disksArgsForCall, struct { @@ -324,34 +400,34 @@ func (fake *FakeEnvironment) DisksCallCount() int { return len(fake.disksArgsForCall) } -func (fake *FakeEnvironment) DisksCalls(stub func() ([]disk.PartitionStat, error)) { +func (fake *FakeEnvironment) DisksCalls(stub func() ([]*proto.DiskPartition, error)) { fake.disksMutex.Lock() defer fake.disksMutex.Unlock() fake.DisksStub = stub } -func (fake *FakeEnvironment) DisksReturns(result1 []disk.PartitionStat, result2 error) { +func (fake *FakeEnvironment) DisksReturns(result1 []*proto.DiskPartition, result2 error) { fake.disksMutex.Lock() defer fake.disksMutex.Unlock() fake.DisksStub = nil fake.disksReturns = struct { - result1 []disk.PartitionStat + result1 []*proto.DiskPartition result2 error }{result1, result2} } -func (fake *FakeEnvironment) DisksReturnsOnCall(i int, result1 []disk.PartitionStat, result2 error) { +func (fake *FakeEnvironment) DisksReturnsOnCall(i int, result1 []*proto.DiskPartition, result2 error) { fake.disksMutex.Lock() defer fake.disksMutex.Unlock() fake.DisksStub = nil if fake.disksReturnsOnCall == nil { fake.disksReturnsOnCall = make(map[int]struct { - result1 []disk.PartitionStat + result1 []*proto.DiskPartition result2 error }) } fake.disksReturnsOnCall[i] = struct { - result1 []disk.PartitionStat + result1 []*proto.DiskPartition result2 error }{result1, result2} } @@ -1012,6 +1088,8 @@ func (fake *FakeEnvironment) Invocations() map[string][][]interface{} { defer fake.deleteFileMutex.RUnlock() fake.diskDevicesMutex.RLock() defer fake.diskDevicesMutex.RUnlock() + fake.diskUsageMutex.RLock() + defer fake.diskUsageMutex.RUnlock() fake.disksMutex.RLock() defer fake.disksMutex.RUnlock() fake.fileStatMutex.RLock() diff --git a/src/core/metrics/sources/disk.go b/src/core/metrics/sources/disk.go index 5b72ddb95..472133097 100644 --- a/src/core/metrics/sources/disk.go +++ b/src/core/metrics/sources/disk.go @@ -15,7 +15,6 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" - "github.com/shirou/gopsutil/v3/disk" ) const MOUNT_POINT = "mount_point" @@ -23,23 +22,24 @@ const MOUNT_POINT = "mount_point" type Disk struct { logger *MetricSourceLogger *namedMetric - disks []disk.PartitionStat + disks []*proto.DiskPartition + env core.Environment } func NewDiskSource(namespace string, env core.Environment) *Disk { disks, _ := env.Disks() - return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} + return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks, env} } func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() for _, part := range c.disks { - if part.Device == "" || part.Fstype == "" { + if part.Device == "" || part.FsType == "" { continue } - usage, err := disk.UsageWithContext(ctx, part.Mountpoint) + usage, err := c.env.DiskUsage(part.MountPoint) if err != nil { - c.logger.Log(fmt.Sprintf("Failed to get disk metrics for mount point %s, %v", part.Mountpoint, err)) + c.logger.Log(fmt.Sprintf("Failed to get disk metrics for mount point %s, %v", part.MountPoint, err)) continue } @@ -47,14 +47,14 @@ func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metric "total": float64(usage.Total), "used": float64(usage.Used), "free": float64(usage.Free), - "in_use": float64(usage.UsedPercent), + "in_use": float64(usage.UsedPercentage), }) select { case <-ctx.Done(): return // mount point is not a common dim - case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.MountPoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } } diff --git a/src/core/metrics/sources/disk_test.go b/src/core/metrics/sources/disk_test.go index 65e4e3bc0..9e0047c03 100644 --- a/src/core/metrics/sources/disk_test.go +++ b/src/core/metrics/sources/disk_test.go @@ -26,7 +26,7 @@ func TestNewDiskSource(t *testing.T) { assert.Equal(t, "disk", actual.group) assert.Equal(t, namespace, actual.namespace) - assert.Greater(t, len(actual.disks), 1) + assert.Equal(t, len(actual.disks), 2) } func TestDiskCollect(t *testing.T) { diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index fe568901a..57ca0209c 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -53,8 +53,9 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) - Disks() ([]disk.PartitionStat, error) + Disks() ([]*proto.DiskPartition, error) DiskDevices() ([]string, error) + DiskUsage(mountPoint string) (*DiskUsage, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) IsContainer() bool @@ -82,6 +83,13 @@ type Process struct { Command string } +type DiskUsage struct { + Total float64 + Used float64 + Free float64 + UsedPercentage float64 +} + const ( lengthOfContainerId = 64 versionId = "VERSION_ID" @@ -136,7 +144,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer OsType: hostInformation.OS, Uuid: env.GetSystemUUID(), Uname: getUnixName(), - Partitons: diskPartitions(), + Partitons: env.Disks(), Network: env.networks(), Processor: processors(hostInformation.KernelArch), Release: releaseInfo("/etc/os-release"), @@ -199,7 +207,7 @@ func (env *EnvironmentType) GetHostname() string { } func (env *EnvironmentType) GetSystemUUID() string { - res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { var err error ctx := context.Background() defer ctx.Done() @@ -219,6 +227,11 @@ func (env *EnvironmentType) GetSystemUUID() string { } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) + if err != nil { + log.Warnf("unable to set hostname due to %v", err) + return "" + } + return res.(string) } @@ -443,11 +456,40 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } -func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { +func (env *EnvironmentType) Disks() (partitions []*proto.DiskPartition, err error) { ctx := context.Background() defer ctx.Done() - disks, err := disk.PartitionsWithContext(ctx, false) - return disks, err + parts, err := disk.PartitionsWithContext(ctx, false) + if err != nil { + // return an array of 0 + log.Errorf("Could not read disk partitions for host: %v", err) + return []*proto.DiskPartition{}, err + } + for _, part := range parts { + pm := proto.DiskPartition{ + MountPoint: part.Mountpoint, + Device: part.Device, + FsType: part.Fstype, + } + partitions = append(partitions, &pm) + } + return partitions, nil +} + +func (env *EnvironmentType) DiskUsage(mountPoint string) (*DiskUsage, error) { + ctx := context.Background() + defer ctx.Done() + usage, err := disk.UsageWithContext(ctx, mountPoint) + if err != nil { + return nil, errors.New("unable to obtain disk usage stats") + } + + return &DiskUsage{ + Total: float64(usage.Total), + Used: float64(usage.Used), + Free: float64(usage.Free), + UsedPercentage: float64(usage.UsedPercent), + }, nil } func (env *EnvironmentType) GetNetOverflow() (float64, error) { @@ -817,26 +859,6 @@ func virtualization() (string, string) { return virtualizationSystem, virtualizationRole } -func diskPartitions() (partitions []*proto.DiskPartition) { - ctx := context.Background() - defer ctx.Done() - parts, err := disk.PartitionsWithContext(ctx, false) - if err != nil { - // return an array of 0 - log.Errorf("Could not read disk partitions for host: %v", err) - return []*proto.DiskPartition{} - } - for _, part := range parts { - pm := proto.DiskPartition{ - MountPoint: part.Mountpoint, - Device: part.Device, - FsType: part.Fstype, - } - partitions = append(partitions, &pm) - } - return partitions -} - func releaseInfo(osReleaseFile string) (release *proto.ReleaseInfo) { hostReleaseInfo := getHostReleaseInfo() osRelease, err := getOsRelease(osReleaseFile) diff --git a/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go index 845b28124..2039a2ce2 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/test/utils/environment.go @@ -5,7 +5,6 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" - "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -14,14 +13,27 @@ type MockEnvironment struct { } // Disks implements core.Environment. -func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { - return []disk.PartitionStat{ +func (*MockEnvironment) Disks() ([]*proto.DiskPartition, error) { + return []*proto.DiskPartition{ { Device: "sd01", - Mountpoint: "/", - Fstype: "ext4", - Opts: []string{"ro"}, + MountPoint: "/sd01", + FsType: "ext4", }, + { + Device: "sd02", + MountPoint: "/sd02", + FsType: "ext4", + }, + }, nil +} + +func (*MockEnvironment) DiskUsage(mountpoint string) (*core.DiskUsage, error) { + return &core.DiskUsage{ + Total: 20, + Used: 10, + Free: 10, + UsedPercentage: 100, }, nil } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index fe568901a..57ca0209c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -53,8 +53,9 @@ type Environment interface { DeleteFile(backup ConfigApplyMarker, fileName string) error Processes() (result []*Process) FileStat(path string) (os.FileInfo, error) - Disks() ([]disk.PartitionStat, error) + Disks() ([]*proto.DiskPartition, error) DiskDevices() ([]string, error) + DiskUsage(mountPoint string) (*DiskUsage, error) GetContainerID() (string, error) GetNetOverflow() (float64, error) IsContainer() bool @@ -82,6 +83,13 @@ type Process struct { Command string } +type DiskUsage struct { + Total float64 + Used float64 + Free float64 + UsedPercentage float64 +} + const ( lengthOfContainerId = 64 versionId = "VERSION_ID" @@ -136,7 +144,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer OsType: hostInformation.OS, Uuid: env.GetSystemUUID(), Uname: getUnixName(), - Partitons: diskPartitions(), + Partitons: env.Disks(), Network: env.networks(), Processor: processors(hostInformation.KernelArch), Release: releaseInfo("/etc/os-release"), @@ -199,7 +207,7 @@ func (env *EnvironmentType) GetHostname() string { } func (env *EnvironmentType) GetSystemUUID() string { - res, _, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) { var err error ctx := context.Background() defer ctx.Done() @@ -219,6 +227,11 @@ func (env *EnvironmentType) GetSystemUUID() string { } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) + if err != nil { + log.Warnf("unable to set hostname due to %v", err) + return "" + } + return res.(string) } @@ -443,11 +456,40 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) { } } -func (env *EnvironmentType) Disks() ([]disk.PartitionStat, error) { +func (env *EnvironmentType) Disks() (partitions []*proto.DiskPartition, err error) { ctx := context.Background() defer ctx.Done() - disks, err := disk.PartitionsWithContext(ctx, false) - return disks, err + parts, err := disk.PartitionsWithContext(ctx, false) + if err != nil { + // return an array of 0 + log.Errorf("Could not read disk partitions for host: %v", err) + return []*proto.DiskPartition{}, err + } + for _, part := range parts { + pm := proto.DiskPartition{ + MountPoint: part.Mountpoint, + Device: part.Device, + FsType: part.Fstype, + } + partitions = append(partitions, &pm) + } + return partitions, nil +} + +func (env *EnvironmentType) DiskUsage(mountPoint string) (*DiskUsage, error) { + ctx := context.Background() + defer ctx.Done() + usage, err := disk.UsageWithContext(ctx, mountPoint) + if err != nil { + return nil, errors.New("unable to obtain disk usage stats") + } + + return &DiskUsage{ + Total: float64(usage.Total), + Used: float64(usage.Used), + Free: float64(usage.Free), + UsedPercentage: float64(usage.UsedPercent), + }, nil } func (env *EnvironmentType) GetNetOverflow() (float64, error) { @@ -817,26 +859,6 @@ func virtualization() (string, string) { return virtualizationSystem, virtualizationRole } -func diskPartitions() (partitions []*proto.DiskPartition) { - ctx := context.Background() - defer ctx.Done() - parts, err := disk.PartitionsWithContext(ctx, false) - if err != nil { - // return an array of 0 - log.Errorf("Could not read disk partitions for host: %v", err) - return []*proto.DiskPartition{} - } - for _, part := range parts { - pm := proto.DiskPartition{ - MountPoint: part.Mountpoint, - Device: part.Device, - FsType: part.Fstype, - } - partitions = append(partitions, &pm) - } - return partitions -} - func releaseInfo(osReleaseFile string) (release *proto.ReleaseInfo) { hostReleaseInfo := getHostReleaseInfo() osRelease, err := getOsRelease(osReleaseFile) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go index 5b72ddb95..472133097 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go @@ -15,7 +15,6 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" - "github.com/shirou/gopsutil/v3/disk" ) const MOUNT_POINT = "mount_point" @@ -23,23 +22,24 @@ const MOUNT_POINT = "mount_point" type Disk struct { logger *MetricSourceLogger *namedMetric - disks []disk.PartitionStat + disks []*proto.DiskPartition + env core.Environment } func NewDiskSource(namespace string, env core.Environment) *Disk { disks, _ := env.Disks() - return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} + return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks, env} } func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() for _, part := range c.disks { - if part.Device == "" || part.Fstype == "" { + if part.Device == "" || part.FsType == "" { continue } - usage, err := disk.UsageWithContext(ctx, part.Mountpoint) + usage, err := c.env.DiskUsage(part.MountPoint) if err != nil { - c.logger.Log(fmt.Sprintf("Failed to get disk metrics for mount point %s, %v", part.Mountpoint, err)) + c.logger.Log(fmt.Sprintf("Failed to get disk metrics for mount point %s, %v", part.MountPoint, err)) continue } @@ -47,14 +47,14 @@ func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metric "total": float64(usage.Total), "used": float64(usage.Used), "free": float64(usage.Free), - "in_use": float64(usage.UsedPercent), + "in_use": float64(usage.UsedPercentage), }) select { case <-ctx.Done(): return // mount point is not a common dim - case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.MountPoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go index 845b28124..2039a2ce2 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go @@ -5,7 +5,6 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" - "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -14,14 +13,27 @@ type MockEnvironment struct { } // Disks implements core.Environment. -func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { - return []disk.PartitionStat{ +func (*MockEnvironment) Disks() ([]*proto.DiskPartition, error) { + return []*proto.DiskPartition{ { Device: "sd01", - Mountpoint: "/", - Fstype: "ext4", - Opts: []string{"ro"}, + MountPoint: "/sd01", + FsType: "ext4", }, + { + Device: "sd02", + MountPoint: "/sd02", + FsType: "ext4", + }, + }, nil +} + +func (*MockEnvironment) DiskUsage(mountpoint string) (*core.DiskUsage, error) { + return &core.DiskUsage{ + Total: 20, + Used: 10, + Free: 10, + UsedPercentage: 100, }, nil } diff --git a/test/utils/environment.go b/test/utils/environment.go index 845b28124..2039a2ce2 100644 --- a/test/utils/environment.go +++ b/test/utils/environment.go @@ -5,7 +5,6 @@ import ( "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" - "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/mock" ) @@ -14,14 +13,27 @@ type MockEnvironment struct { } // Disks implements core.Environment. -func (*MockEnvironment) Disks() ([]disk.PartitionStat, error) { - return []disk.PartitionStat{ +func (*MockEnvironment) Disks() ([]*proto.DiskPartition, error) { + return []*proto.DiskPartition{ { Device: "sd01", - Mountpoint: "/", - Fstype: "ext4", - Opts: []string{"ro"}, + MountPoint: "/sd01", + FsType: "ext4", }, + { + Device: "sd02", + MountPoint: "/sd02", + FsType: "ext4", + }, + }, nil +} + +func (*MockEnvironment) DiskUsage(mountpoint string) (*core.DiskUsage, error) { + return &core.DiskUsage{ + Total: 20, + Used: 10, + Free: 10, + UsedPercentage: 100, }, nil } From c6bcab2faac2dae7cbee262375665c80dee4d7b6 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Mon, 25 Sep 2023 10:45:10 +0100 Subject: [PATCH 10/14] local changes --- src/core/environment.go | 1 + .../github.com/nginx/agent/v2/src/core/environment.go | 8 +++++++- .../github.com/nginx/agent/v2/src/core/environment.go | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 080c17f31..6ce7bd5da 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -141,6 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 57ca0209c..6ce7bd5da 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -136,6 +136,12 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer tags = &[]string{} } + disks, err := env.Disks() + if err != nil { + log.Warnf("Unable to get disks information from the host: %v", err) + disks = nil + } + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -144,7 +150,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer OsType: hostInformation.OS, Uuid: env.GetSystemUUID(), Uname: getUnixName(), - Partitons: env.Disks(), + Partitons: disks, Network: env.networks(), Processor: processors(hostInformation.KernelArch), Release: releaseInfo("/etc/os-release"), diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 57ca0209c..6ce7bd5da 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -136,6 +136,12 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer tags = &[]string{} } + disks, err := env.Disks() + if err != nil { + log.Warnf("Unable to get disks information from the host: %v", err) + disks = nil + } + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -144,7 +150,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer OsType: hostInformation.OS, Uuid: env.GetSystemUUID(), Uname: getUnixName(), - Partitons: env.Disks(), + Partitons: disks, Network: env.networks(), Processor: processors(hostInformation.KernelArch), Release: releaseInfo("/etc/os-release"), From 94815806e0cb465915054feeef51955bf152db70 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Tue, 26 Sep 2023 09:52:12 +0100 Subject: [PATCH 11/14] handling error --- src/core/environment.go | 8 ++++++-- .../github.com/nginx/agent/v2/src/core/environment.go | 8 ++++++-- .../github.com/nginx/agent/v2/src/core/environment.go | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 6ce7bd5da..d35bb4e35 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -334,7 +334,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -348,6 +348,10 @@ func (env *EnvironmentType) IsContainer() bool { return false, nil }) + if err != nil { + log.Warnf("unable to retrieve values from cache (%v)", err) + } + return res.(bool) } diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 6ce7bd5da..d35bb4e35 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -334,7 +334,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -348,6 +348,10 @@ func (env *EnvironmentType) IsContainer() bool { return false, nil }) + if err != nil { + log.Warnf("unable to retrieve values from cache (%v)", err) + } + return res.(bool) } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 6ce7bd5da..d35bb4e35 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, @@ -334,7 +334,7 @@ func (env *EnvironmentType) IsContainer() bool { k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount" ) - res, _, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { + res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { log.Debugf("is a container because (%s) exists", filename) @@ -348,6 +348,10 @@ func (env *EnvironmentType) IsContainer() bool { return false, nil }) + if err != nil { + log.Warnf("unable to retrieve values from cache (%v)", err) + } + return res.(bool) } From 3e703904c8c3bfd025f7c2f22c9a290be0e3b3a7 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Tue, 26 Sep 2023 09:52:52 +0100 Subject: [PATCH 12/14] handling error --- src/core/environment.go | 2 +- .../vendor/github.com/nginx/agent/v2/src/core/environment.go | 2 +- .../vendor/github.com/nginx/agent/v2/src/core/environment.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index d35bb4e35..6ded07a34 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index d35bb4e35..6ded07a34 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index d35bb4e35..6ded07a34 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -141,7 +141,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer log.Warnf("Unable to get disks information from the host: %v", err) disks = nil } - + hostInfoFacacde := &proto.HostInfo{ Agent: agentVersion, Boot: hostInformation.BootTime, From 8a3db7ba1a6340c11379cf5ad9ce1df34e80039c Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Tue, 26 Sep 2023 13:10:32 +0100 Subject: [PATCH 13/14] updated log statements to match convention --- src/core/environment.go | 8 ++++---- .../github.com/nginx/agent/v2/src/core/environment.go | 8 ++++---- .../github.com/nginx/agent/v2/src/core/environment.go | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core/environment.go b/src/core/environment.go index 6ded07a34..b80a0112d 100644 --- a/src/core/environment.go +++ b/src/core/environment.go @@ -228,13 +228,13 @@ func (env *EnvironmentType) GetSystemUUID() string { hostID, err := host.HostIDWithContext(ctx) if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) + log.Warnf("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "", err } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) if err != nil { - log.Warnf("unable to set hostname due to %v", err) + log.Warnf("Unable to set hostname due to %v", err) return "" } @@ -337,7 +337,7 @@ func (env *EnvironmentType) IsContainer() bool { res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) + log.Debugf("Is a container because (%s) exists", filename) return true, nil } } @@ -349,7 +349,7 @@ func (env *EnvironmentType) IsContainer() bool { }) if err != nil { - log.Warnf("unable to retrieve values from cache (%v)", err) + log.Warnf("Unable to retrieve values from cache (%v)", err) } return res.(bool) diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go index 6ded07a34..b80a0112d 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -228,13 +228,13 @@ func (env *EnvironmentType) GetSystemUUID() string { hostID, err := host.HostIDWithContext(ctx) if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) + log.Warnf("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "", err } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) if err != nil { - log.Warnf("unable to set hostname due to %v", err) + log.Warnf("Unable to set hostname due to %v", err) return "" } @@ -337,7 +337,7 @@ func (env *EnvironmentType) IsContainer() bool { res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) + log.Debugf("Is a container because (%s) exists", filename) return true, nil } } @@ -349,7 +349,7 @@ func (env *EnvironmentType) IsContainer() bool { }) if err != nil { - log.Warnf("unable to retrieve values from cache (%v)", err) + log.Warnf("Unable to retrieve values from cache (%v)", err) } return res.(bool) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go index 6ded07a34..b80a0112d 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/environment.go @@ -228,13 +228,13 @@ func (env *EnvironmentType) GetSystemUUID() string { hostID, err := host.HostIDWithContext(ctx) if err != nil { - log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err) + log.Warnf("Unable to read host id from dataplane, defaulting value. Error: %v", err) return "", err } return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err }) if err != nil { - log.Warnf("unable to set hostname due to %v", err) + log.Warnf("Unable to set hostname due to %v", err) return "" } @@ -337,7 +337,7 @@ func (env *EnvironmentType) IsContainer() bool { res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) { for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} { if _, err := os.Stat(filename); err == nil { - log.Debugf("is a container because (%s) exists", filename) + log.Debugf("Is a container because (%s) exists", filename) return true, nil } } @@ -349,7 +349,7 @@ func (env *EnvironmentType) IsContainer() bool { }) if err != nil { - log.Warnf("unable to retrieve values from cache (%v)", err) + log.Warnf("Unable to retrieve values from cache (%v)", err) } return res.(bool) From 591a537c67d97ee8ebd924305a2d0989f1eb4c78 Mon Sep 17 00:00:00 2001 From: Oliver O'Mahony Date: Thu, 28 Sep 2023 14:53:12 +0100 Subject: [PATCH 14/14] merge from main --- src/core/nginx.go | 1 - .../vendor/github.com/nginx/agent/v2/src/core/nginx.go | 1 - .../vendor/github.com/nginx/agent/v2/src/core/nginx.go | 1 - 3 files changed, 3 deletions(-) diff --git a/src/core/nginx.go b/src/core/nginx.go index 04dcc944e..f29f410ee 100644 --- a/src/core/nginx.go +++ b/src/core/nginx.go @@ -131,7 +131,6 @@ func (n *NginxBinaryType) UpdateNginxDetailsFromProcesses(nginxProcesses []*Proc n.nginxWorkersMap[nginxDetails.GetNginxId()] = append(n.nginxWorkersMap[nginxDetails.GetNginxId()], nginxDetails) } } - } func (n *NginxBinaryType) GetChildProcesses() map[string][]*proto.NginxDetails { diff --git a/test/integration/vendor/github.com/nginx/agent/v2/src/core/nginx.go b/test/integration/vendor/github.com/nginx/agent/v2/src/core/nginx.go index 04dcc944e..f29f410ee 100644 --- a/test/integration/vendor/github.com/nginx/agent/v2/src/core/nginx.go +++ b/test/integration/vendor/github.com/nginx/agent/v2/src/core/nginx.go @@ -131,7 +131,6 @@ func (n *NginxBinaryType) UpdateNginxDetailsFromProcesses(nginxProcesses []*Proc n.nginxWorkersMap[nginxDetails.GetNginxId()] = append(n.nginxWorkersMap[nginxDetails.GetNginxId()], nginxDetails) } } - } func (n *NginxBinaryType) GetChildProcesses() map[string][]*proto.NginxDetails { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/nginx.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/nginx.go index 04dcc944e..f29f410ee 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/nginx.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/nginx.go @@ -131,7 +131,6 @@ func (n *NginxBinaryType) UpdateNginxDetailsFromProcesses(nginxProcesses []*Proc n.nginxWorkersMap[nginxDetails.GetNginxId()] = append(n.nginxWorkersMap[nginxDetails.GetNginxId()], nginxDetails) } } - } func (n *NginxBinaryType) GetChildProcesses() map[string][]*proto.NginxDetails {