Skip to content

Commit

Permalink
Added non-windows virtual memory scraper implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
james-bebbington committed May 25, 2020
1 parent 19d8b43 commit 9d70736
Show file tree
Hide file tree
Showing 10 changed files with 305 additions and 67 deletions.
30 changes: 14 additions & 16 deletions receiver/hostmetricsreceiver/hostmetrics_receiver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,16 @@ var standardMetrics = []string{
"host/network/errors",
"host/network/bytes",
"host/network/tcp_connections",
"host/swap/paging",
"host/swap/usage",
}

var systemSpecificMetrics = map[string][]string{
"linux": {"host/filesystem/inodes/used"},
"darwin": {"host/filesystem/inodes/used"},
"freebsd": {"host/filesystem/inodes/used"},
"openbsd": {"host/filesystem/inodes/used"},
"solaris": {"host/filesystem/inodes/used"},
"windows": {"host/swap/usage", "host/swap/paging"},
"linux": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"darwin": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"freebsd": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"openbsd": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"solaris": {"host/filesystem/inodes/used", "host/swap/page_faults"},
}

func TestGatherMetrics_EndToEnd(t *testing.T) {
Expand All @@ -70,19 +71,16 @@ func TestGatherMetrics_EndToEnd(t *testing.T) {
config := &Config{
CollectionInterval: 100 * time.Millisecond,
Scrapers: map[string]internal.Config{
cpuscraper.TypeStr: &cpuscraper.Config{ReportPerCPU: true},
diskscraper.TypeStr: &diskscraper.Config{},
filesystemscraper.TypeStr: &filesystemscraper.Config{},
loadscraper.TypeStr: &loadscraper.Config{},
memoryscraper.TypeStr: &memoryscraper.Config{},
networkscraper.TypeStr: &networkscraper.Config{},
cpuscraper.TypeStr: &cpuscraper.Config{ReportPerCPU: true},
diskscraper.TypeStr: &diskscraper.Config{},
filesystemscraper.TypeStr: &filesystemscraper.Config{},
loadscraper.TypeStr: &loadscraper.Config{},
memoryscraper.TypeStr: &memoryscraper.Config{},
networkscraper.TypeStr: &networkscraper.Config{},
virtualmemoryscraper.TypeStr: &virtualmemoryscraper.Config{},
},
}

if runtime.GOOS == "windows" {
config.Scrapers[virtualmemoryscraper.TypeStr] = &virtualmemoryscraper.Config{}
}

factories := map[string]internal.Factory{
cpuscraper.TypeStr: &cpuscraper.Factory{},
diskscraper.TypeStr: &diskscraper.Factory{},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ package virtualmemoryscraper

import (
"context"
"errors"
"runtime"

"go.uber.org/zap"

Expand Down Expand Up @@ -46,10 +44,6 @@ func (f *Factory) CreateMetricsScraper(
logger *zap.Logger,
config internal.Config,
) (internal.Scraper, error) {
if runtime.GOOS != "windows" {
return nil, errors.New("the virtual memory scraper is currently only supported on windows")
}

cfg := config.(*Config)
return newVirtualMemoryScraper(ctx, cfg), nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package virtualmemoryscraper

import (
"context"
"runtime"
"testing"

"github.com/stretchr/testify/assert"
Expand All @@ -34,12 +33,6 @@ func TestCreateMetricsScraper(t *testing.T) {
cfg := &Config{}

scraper, err := factory.CreateMetricsScraper(context.Background(), zap.NewNop(), cfg)

if runtime.GOOS == "windows" {
assert.NoError(t, err)
assert.NotNil(t, scraper)
} else {
assert.Error(t, err)
assert.Nil(t, scraper)
}
assert.NoError(t, err)
assert.NotNil(t, scraper)
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ type enumPageFileInformation struct {
peakUsage uint64
}

var getPageFileStats = getPageFileStatsInternal

func getPageFileStatsInternal() ([]*pageFileData, error) {
// the following system call invokes the supplied callback function once for each page file before returning
// see https://docs.microsoft.com/en-us/windows/win32/api/psapi/nf-psapi-enumpagefilesw
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,22 @@ import (
// virtual memory metric constants

const (
deviceLabelName = "device"
directionLabelName = "direction"
pageFileLabelName = "pagefile"
stateLabelName = "state"
typeLabelName = "type"
)

const (
majorTypeLabelValue = "major"
majorTypeLabelValue = "major"
minorTypeLabelValue = "minor"

inDirectionLabelValue = "page_in"
outDirectionLabelValue = "page_out"
usedLabelValue = "used"
freeLabelValue = "free"

cachedLabelValue = "cached"
freeLabelValue = "free"
usedLabelValue = "used"
)

var metricSwapUsageDescriptor = createMetricSwapUsageDescriptor()
Expand All @@ -58,3 +62,15 @@ func createMetricPagingDescriptor() pdata.MetricDescriptor {
descriptor.SetType(pdata.MetricTypeCounterInt64)
return descriptor
}

var metricPageFaultsDescriptor = createMetricPageFaultsDescriptor()

func createMetricPageFaultsDescriptor() pdata.MetricDescriptor {
descriptor := pdata.NewMetricDescriptor()
descriptor.InitEmpty()
descriptor.SetName("host/swap/page_faults")
descriptor.SetDescription("The number of page faults.")
descriptor.SetUnit("1")
descriptor.SetType(pdata.MetricTypeCounterInt64)
return descriptor
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,20 @@ package virtualmemoryscraper

import (
"context"
"time"

"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"go.opencensus.io/trace"

"go.opentelemetry.io/collector/component/componenterror"
"go.opentelemetry.io/collector/consumer/pdata"
)

// scraper for VirtualMemory Metrics
type scraper struct {
config *Config
config *Config
startTime pdata.TimestampUnixNano
}

// newVirtualMemoryScraper creates a VirtualMemory Scraper
Expand All @@ -34,6 +41,12 @@ func newVirtualMemoryScraper(_ context.Context, cfg *Config) *scraper {

// Initialize
func (s *scraper) Initialize(_ context.Context) error {
bootTime, err := host.BootTime()
if err != nil {
return err
}

s.startTime = pdata.TimestampUnixNano(bootTime)
return nil
}

Expand All @@ -44,5 +57,108 @@ func (s *scraper) Close(_ context.Context) error {

// ScrapeMetrics
func (s *scraper) ScrapeMetrics(ctx context.Context) (pdata.MetricSlice, error) {
return pdata.NewMetricSlice(), nil
_, span := trace.StartSpan(ctx, "virtualmemoryscraper.ScrapeMetrics")
defer span.End()

metrics := pdata.NewMetricSlice()

var errors []error

err := s.scrapeAndAppendSwapUsageMetric(metrics)
if err != nil {
errors = append(errors, err)
}

err = s.scrapeAndAppendPagingMetrics(metrics)
if err != nil {
errors = append(errors, err)
}

if len(errors) > 0 {
return metrics, componenterror.CombineErrors(errors)
}

return metrics, nil
}

var getVirtualMemory = mem.VirtualMemory

func (s *scraper) scrapeAndAppendSwapUsageMetric(metrics pdata.MetricSlice) error {
vmem, err := getVirtualMemory()
if err != nil {
return err
}

idx := metrics.Len()
metrics.Resize(idx + 1)
initializeSwapUsageMetric(metrics.At(idx), vmem)
return nil
}

func initializeSwapUsageMetric(metric pdata.Metric, vmem *mem.VirtualMemoryStat) {
metricSwapUsageDescriptor.CopyTo(metric.MetricDescriptor())

idps := metric.Int64DataPoints()
idps.Resize(3)
initializeSwapUsageDataPoint(idps.At(0), usedLabelValue, int64(vmem.SwapTotal-vmem.SwapFree-vmem.SwapCached))
initializeSwapUsageDataPoint(idps.At(1), freeLabelValue, int64(vmem.SwapFree))
initializeSwapUsageDataPoint(idps.At(2), cachedLabelValue, int64(vmem.SwapCached))
}

func initializeSwapUsageDataPoint(dataPoint pdata.Int64DataPoint, stateLabel string, value int64) {
labelsMap := dataPoint.LabelsMap()
labelsMap.Insert(stateLabelName, stateLabel)
dataPoint.SetTimestamp(pdata.TimestampUnixNano(uint64(time.Now().UnixNano())))
dataPoint.SetValue(value)
}

var getSwapMemory = mem.SwapMemory

func (s *scraper) scrapeAndAppendPagingMetrics(metrics pdata.MetricSlice) error {
swap, err := getSwapMemory()
if err != nil {
return err
}

idx := metrics.Len()
metrics.Resize(idx + 2)
initializePagingMetric(metrics.At(idx+0), s.startTime, swap)
initializePageFaultsMetric(metrics.At(idx+1), s.startTime, swap)
return nil
}

func initializePagingMetric(metric pdata.Metric, startTime pdata.TimestampUnixNano, swap *mem.SwapMemoryStat) {
metricPagingDescriptor.CopyTo(metric.MetricDescriptor())

idps := metric.Int64DataPoints()
idps.Resize(4)
initializePagingDataPoint(idps.At(0), startTime, majorTypeLabelValue, inDirectionLabelValue, int64(swap.Sin))
initializePagingDataPoint(idps.At(1), startTime, majorTypeLabelValue, outDirectionLabelValue, int64(swap.Sout))
initializePagingDataPoint(idps.At(2), startTime, minorTypeLabelValue, inDirectionLabelValue, int64(swap.PgIn))
initializePagingDataPoint(idps.At(3), startTime, minorTypeLabelValue, outDirectionLabelValue, int64(swap.PgOut))
}

func initializePagingDataPoint(dataPoint pdata.Int64DataPoint, startTime pdata.TimestampUnixNano, typeLabel string, directionLabel string, value int64) {
labelsMap := dataPoint.LabelsMap()
labelsMap.Insert(typeLabelName, typeLabel)
labelsMap.Insert(directionLabelName, directionLabel)
dataPoint.SetStartTime(startTime)
dataPoint.SetTimestamp(pdata.TimestampUnixNano(uint64(time.Now().UnixNano())))
dataPoint.SetValue(value)
}

func initializePageFaultsMetric(metric pdata.Metric, startTime pdata.TimestampUnixNano, swap *mem.SwapMemoryStat) {
metricPageFaultsDescriptor.CopyTo(metric.MetricDescriptor())

idps := metric.Int64DataPoints()
idps.Resize(1)
initializePageFaultDataPoint(idps.At(0), startTime, minorTypeLabelValue, int64(swap.PgFault))
// TODO add swap.PgMajFault once available in gopsutil
}

func initializePageFaultDataPoint(dataPoint pdata.Int64DataPoint, startTime pdata.TimestampUnixNano, typeLabel string, value int64) {
dataPoint.LabelsMap().Insert(typeLabelName, typeLabel)
dataPoint.SetStartTime(startTime)
dataPoint.SetTimestamp(pdata.TimestampUnixNano(uint64(time.Now().UnixNano())))
dataPoint.SetValue(value)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !windows

package virtualmemoryscraper

import (
"context"
"errors"
"testing"

"github.com/shirou/gopsutil/mem"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestScrapeMetrics_Errors(t *testing.T) {
type testCase struct {
name string
virtualMemoryError error
swapMemoryError error
expectedError string
}

testCases := []testCase{
{
name: "virtualMemoryError",
virtualMemoryError: errors.New("err1"),
expectedError: "err1",
},
{
name: "swapMemoryError",
swapMemoryError: errors.New("err2"),
expectedError: "err2",
},
{
name: "multipleErrors",
virtualMemoryError: errors.New("err1"),
swapMemoryError: errors.New("err2"),
expectedError: "[err1; err2]",
},
}

preservedGetVirtualMemory := getVirtualMemory
preservedGetSwapMemory := getSwapMemory

scraper := newVirtualMemoryScraper(context.Background(), &Config{})
err := scraper.Initialize(context.Background())
require.NoError(t, err, "Failed to initialize virtual memory scraper: %v", err)
defer func() { assert.NoError(t, scraper.Close(context.Background())) }()

for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
if test.virtualMemoryError != nil {
getVirtualMemory = func() (*mem.VirtualMemoryStat, error) { return nil, test.virtualMemoryError }
}
if test.swapMemoryError != nil {
getSwapMemory = func() (*mem.SwapMemoryStat, error) { return nil, test.swapMemoryError }
}

_, err = scraper.ScrapeMetrics(context.Background())
assert.EqualError(t, err, test.expectedError)

getVirtualMemory = preservedGetVirtualMemory
getSwapMemory = preservedGetSwapMemory
})
}
}

0 comments on commit 9d70736

Please sign in to comment.