Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
emit container cpu/memory/disk metrics periodically
[#76563380] Signed-off-by: Ashvin Agrawal <ashvin@pivotal.io>
- Loading branch information
Showing
7 changed files
with
552 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package containermetrics_test | ||
|
|
||
| import ( | ||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| "testing" | ||
| ) | ||
|
|
||
| func TestContainerMetrics(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
| RunSpecs(t, "ContainerMetrics Suite") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| package containermetrics | ||
|
|
||
| import ( | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/cloudfoundry-incubator/executor" | ||
| "github.com/cloudfoundry/dropsonde/metrics" | ||
| "github.com/pivotal-golang/clock" | ||
| "github.com/pivotal-golang/lager" | ||
| ) | ||
|
|
||
| type StatsReporter struct { | ||
| logger lager.Logger | ||
|
|
||
| interval time.Duration | ||
| clock clock.Clock | ||
| executorClient executor.Client | ||
|
|
||
| cpuInfos map[string]cpuInfo | ||
| } | ||
|
|
||
| type cpuInfo struct { | ||
| timeSpentInCPU time.Duration | ||
| timeOfSample time.Time | ||
| } | ||
|
|
||
| func NewStatsReporter(logger lager.Logger, interval time.Duration, clock clock.Clock, executorClient executor.Client) *StatsReporter { | ||
| return &StatsReporter{ | ||
| logger: logger, | ||
|
|
||
| interval: interval, | ||
| clock: clock, | ||
| executorClient: executorClient, | ||
|
|
||
| cpuInfos: make(map[string]cpuInfo), | ||
| } | ||
| } | ||
|
|
||
| func (reporter *StatsReporter) Run(signals <-chan os.Signal, ready chan<- struct{}) error { | ||
| close(ready) | ||
|
|
||
| ticker := reporter.clock.NewTicker(reporter.interval) | ||
|
|
||
| for { | ||
| select { | ||
| case <-signals: | ||
| return nil | ||
|
|
||
| case <-ticker.C(): | ||
| reporter.emitContainerMetrics(reporter.logger.Session("tick")) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (reporter *StatsReporter) emitContainerMetrics(logger lager.Logger) { | ||
| startTime := reporter.clock.Now() | ||
|
|
||
| logger.Info("started") | ||
| defer func() { | ||
| logger.Info("done", lager.Data{"took": reporter.clock.Now().Sub(startTime)}) | ||
| }() | ||
|
|
||
| containers, err := reporter.executorClient.ListContainers(nil) | ||
| if err != nil { | ||
| logger.Error("failed-to-list-containers", err) | ||
| return | ||
| } | ||
|
|
||
| logger.Info("emitting", lager.Data{ | ||
| "total-containers": len(containers), | ||
| "listing-containers-took": reporter.clock.Now().Sub(startTime), | ||
| }) | ||
|
|
||
| for _, container := range containers { | ||
| if container.MetricsConfig.Guid == "" { | ||
| continue | ||
| } | ||
|
|
||
| currentInfo := cpuInfo{ | ||
| timeSpentInCPU: container.TimeSpentInCPU, | ||
| timeOfSample: reporter.clock.Now(), | ||
| } | ||
|
|
||
| previousInfo, found := reporter.cpuInfos[container.Guid] | ||
|
|
||
| reporter.cpuInfos[container.Guid] = currentInfo | ||
|
|
||
| var cpuPercent float64 | ||
| if !found { | ||
| cpuPercent = 0.0 | ||
| } else { | ||
| cpuPercent = computeCPUPercent( | ||
| previousInfo.timeSpentInCPU, | ||
| currentInfo.timeSpentInCPU, | ||
| previousInfo.timeOfSample, | ||
| currentInfo.timeOfSample, | ||
| ) | ||
| } | ||
|
|
||
| var index int32 | ||
| if container.MetricsConfig.Index != nil { | ||
| index = int32(*container.MetricsConfig.Index) | ||
| } else { | ||
| index = -1 | ||
| } | ||
|
|
||
| err = metrics.SendContainerMetric(container.MetricsConfig.Guid, index, cpuPercent, container.MemoryUsageInBytes, container.DiskUsageInBytes) | ||
| if err != nil { | ||
| logger.Error("failed-to-send-container-metrics", err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // scale from 0 - 100 | ||
| func computeCPUPercent(timeSpentA, timeSpentB time.Duration, sampleTimeA, sampleTimeB time.Time) float64 { | ||
| // divide change in time spent in CPU over time between samples. | ||
| // result is out of 100. | ||
| // | ||
| // don't worry about overflowing int64. it's like, 30 years. | ||
| return float64((timeSpentB-timeSpentA)*100) / float64(sampleTimeB.UnixNano()-sampleTimeA.UnixNano()) | ||
| } |
Oops, something went wrong.