Skip to content

Commit

Permalink
Added virtual memory scraper to hostmetrics receiver (open-telemetry#989
Browse files Browse the repository at this point in the history
)

* Initial commit of host metrics virtual memory scraper for windows

* Added non-windows virtual memory scraper implementation
  • Loading branch information
james-bebbington authored and wyTrivail committed Jul 13, 2020
1 parent 7b3a920 commit 3d83031
Show file tree
Hide file tree
Showing 14 changed files with 958 additions and 40 deletions.
14 changes: 8 additions & 6 deletions receiver/hostmetricsreceiver/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/loadscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/memoryscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/networkscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/virtualmemoryscraper"
)

// This file implements Factory for HostMetrics receiver.
Expand All @@ -52,12 +53,13 @@ type Factory struct {
func NewFactory() *Factory {
return &Factory{
scraperFactories: map[string]internal.Factory{
cpuscraper.TypeStr: &cpuscraper.Factory{},
diskscraper.TypeStr: &diskscraper.Factory{},
loadscraper.TypeStr: &loadscraper.Factory{},
filesystemscraper.TypeStr: &filesystemscraper.Factory{},
memoryscraper.TypeStr: &memoryscraper.Factory{},
networkscraper.TypeStr: &networkscraper.Factory{},
cpuscraper.TypeStr: &cpuscraper.Factory{},
diskscraper.TypeStr: &diskscraper.Factory{},
loadscraper.TypeStr: &loadscraper.Factory{},
filesystemscraper.TypeStr: &filesystemscraper.Factory{},
memoryscraper.TypeStr: &memoryscraper.Factory{},
networkscraper.TypeStr: &networkscraper.Factory{},
virtualmemoryscraper.TypeStr: &virtualmemoryscraper.Factory{},
},
}
}
Expand Down
41 changes: 26 additions & 15 deletions receiver/hostmetricsreceiver/hostmetrics_receiver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ import (
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/cpuscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/diskscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/loadscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/memoryscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/networkscraper"
"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal/scraper/virtualmemoryscraper"
)

var standardMetrics = []string{
Expand All @@ -43,19 +45,24 @@ var standardMetrics = []string{
"host/disk/ops",
"host/disk/time",
"host/filesystem/used",
"host/load/1m",
"host/load/5m",
"host/load/15m",
"host/network/packets",
"host/network/dropped_packets",
"host/network/errors",
"host/network/bytes",
"host/network/tcp_connections",
"host/swap/paging",
"host/swap/usage",
}

var systemSpecificMetrics = map[string][]string{
"linux": {"host/filesystem/inodes/used"},
"darwin": {"host/filesystem/inodes/used"},
"freebsd": {"host/filesystem/inodes/used"},
"openbsd": {"host/filesystem/inodes/used"},
"solaris": {"host/filesystem/inodes/used"},
"linux": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"darwin": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"freebsd": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"openbsd": {"host/filesystem/inodes/used", "host/swap/page_faults"},
"solaris": {"host/filesystem/inodes/used", "host/swap/page_faults"},
}

func TestGatherMetrics_EndToEnd(t *testing.T) {
Expand All @@ -64,20 +71,24 @@ func TestGatherMetrics_EndToEnd(t *testing.T) {
config := &Config{
CollectionInterval: 100 * time.Millisecond,
Scrapers: map[string]internal.Config{
cpuscraper.TypeStr: &cpuscraper.Config{ReportPerCPU: true},
diskscraper.TypeStr: &diskscraper.Config{},
filesystemscraper.TypeStr: &filesystemscraper.Config{},
memoryscraper.TypeStr: &memoryscraper.Config{},
networkscraper.TypeStr: &networkscraper.Config{},
cpuscraper.TypeStr: &cpuscraper.Config{ReportPerCPU: true},
diskscraper.TypeStr: &diskscraper.Config{},
filesystemscraper.TypeStr: &filesystemscraper.Config{},
loadscraper.TypeStr: &loadscraper.Config{},
memoryscraper.TypeStr: &memoryscraper.Config{},
networkscraper.TypeStr: &networkscraper.Config{},
virtualmemoryscraper.TypeStr: &virtualmemoryscraper.Config{},
},
}

factories := map[string]internal.Factory{
cpuscraper.TypeStr: &cpuscraper.Factory{},
diskscraper.TypeStr: &diskscraper.Factory{},
filesystemscraper.TypeStr: &filesystemscraper.Factory{},
memoryscraper.TypeStr: &memoryscraper.Factory{},
networkscraper.TypeStr: &networkscraper.Factory{},
cpuscraper.TypeStr: &cpuscraper.Factory{},
diskscraper.TypeStr: &diskscraper.Factory{},
filesystemscraper.TypeStr: &filesystemscraper.Factory{},
loadscraper.TypeStr: &loadscraper.Factory{},
memoryscraper.TypeStr: &memoryscraper.Factory{},
networkscraper.TypeStr: &networkscraper.Factory{},
virtualmemoryscraper.TypeStr: &virtualmemoryscraper.Factory{},
}

receiver, err := newHostMetricsReceiver(context.Background(), zap.NewNop(), config, factories, sink)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestStartSampling(t *testing.T) {
// override the processor queue length perf counter with a mock
// that will ensure a positive value is returned
assert.IsType(t, &pdh.PerfCounter{}, samplerInstance.processorQueueLengthCounter)
samplerInstance.processorQueueLengthCounter = pdh.NewMockPerfCounter(100)
samplerInstance.processorQueueLengthCounter = pdh.NewMockPerfCounter(100.0)

// second call to startSampling should succeed, but not do anything
startSampling(context.Background(), zap.NewNop())
Expand Down Expand Up @@ -78,24 +78,25 @@ func assertSamplingUnderway(t *testing.T) {
}

func TestSampleLoad(t *testing.T) {
mockCounter := pdh.NewMockPerfCounter(10, 20, 30, 40, 50)
counterReturnValues := []interface{}{10.0, 20.0, 30.0, 40.0, 50.0}
mockCounter := pdh.NewMockPerfCounter(counterReturnValues...)
samplerInstance = &sampler{processorQueueLengthCounter: mockCounter}

for i := 0; i < len(mockCounter.ReturnValues); i++ {
for i := 0; i < len(counterReturnValues); i++ {
samplerInstance.sampleLoad()
}

assert.Equal(t, calcExpectedLoad(mockCounter.ReturnValues, loadAvgFactor1m), samplerInstance.loadAvg1m)
assert.Equal(t, calcExpectedLoad(mockCounter.ReturnValues, loadAvgFactor5m), samplerInstance.loadAvg5m)
assert.Equal(t, calcExpectedLoad(mockCounter.ReturnValues, loadAvgFactor15m), samplerInstance.loadAvg15m)
assert.Equal(t, calcExpectedLoad(counterReturnValues, loadAvgFactor1m), samplerInstance.loadAvg1m)
assert.Equal(t, calcExpectedLoad(counterReturnValues, loadAvgFactor5m), samplerInstance.loadAvg5m)
assert.Equal(t, calcExpectedLoad(counterReturnValues, loadAvgFactor15m), samplerInstance.loadAvg15m)
}

func calcExpectedLoad(scrapedValues []float64, loadAvgFactor float64) float64 {
func calcExpectedLoad(scrapedValues []interface{}, loadAvgFactor float64) float64 {
// replicate the calculations that should be performed to determine the exponentially
// weighted moving averages based on the specified scraped values
var expectedLoad float64
for i := 0; i < len(scrapedValues); i++ {
expectedLoad = expectedLoad*loadAvgFactor + scrapedValues[i]*(1-loadAvgFactor)
expectedLoad = expectedLoad*loadAvgFactor + scrapedValues[i].(float64)*(1-loadAvgFactor)
}
return expectedLoad
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package virtualmemoryscraper

import "go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal"

// Config relating to VirtualMemory Metric Scraper.
type Config struct {
internal.ConfigSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package virtualmemoryscraper

import (
"context"

"go.uber.org/zap"

"go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal"
)

// This file implements Factory for VirtualMemory scraper.

const (
// The value of "type" key in configuration.
TypeStr = "virtualmemory"
)

// Factory is the Factory for scraper.
type Factory struct {
}

// CreateDefaultConfig creates the default configuration for the Scraper.
func (f *Factory) CreateDefaultConfig() internal.Config {
return &Config{}
}

// CreateMetricsScraper creates a scraper based on provided config.
func (f *Factory) CreateMetricsScraper(
ctx context.Context,
logger *zap.Logger,
config internal.Config,
) (internal.Scraper, error) {
cfg := config.(*Config)
return newVirtualMemoryScraper(ctx, cfg), nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package virtualmemoryscraper

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
"go.uber.org/zap"
)

func TestCreateDefaultConfig(t *testing.T) {
factory := &Factory{}
cfg := factory.CreateDefaultConfig()
assert.IsType(t, &Config{}, cfg)
}

func TestCreateMetricsScraper(t *testing.T) {
factory := &Factory{}
cfg := &Config{}

scraper, err := factory.CreateMetricsScraper(context.Background(), zap.NewNop(), cfg)
assert.NoError(t, err)
assert.NotNil(t, scraper)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build windows

package virtualmemoryscraper

import (
"syscall"
"unsafe"

"golang.org/x/sys/windows"
)

var (
modPsapi = windows.NewLazySystemDLL("psapi.dll")
procEnumPageFilesW = modPsapi.NewProc("EnumPageFilesW")
)

type pageFileData struct {
name string
used uint64
total uint64
}

// system type as defined in https://docs.microsoft.com/en-us/windows/win32/api/psapi/ns-psapi-enum_page_file_information
type enumPageFileInformation struct {
cb uint32
reserved uint32
totalSize uint64
totalInUse uint64
peakUsage uint64
}

func getPageFileStatsInternal() ([]*pageFileData, error) {
// the following system call invokes the supplied callback function once for each page file before returning
// see https://docs.microsoft.com/en-us/windows/win32/api/psapi/nf-psapi-enumpagefilesw
var pageFiles []*pageFileData
result, _, _ := procEnumPageFilesW.Call(windows.NewCallback(pEnumPageFileCallbackW), uintptr(unsafe.Pointer(&pageFiles)))
if result == 0 {
return nil, windows.GetLastError()
}

return pageFiles, nil
}

// system callback as defined in https://docs.microsoft.com/en-us/windows/win32/api/psapi/nc-psapi-penum_page_file_callbackw
func pEnumPageFileCallbackW(pageFiles *[]*pageFileData, enumPageFileInfo *enumPageFileInformation, lpFilenamePtr *[syscall.MAX_LONG_PATH]uint16) *bool {
pageFileName := syscall.UTF16ToString((*lpFilenamePtr)[:])

pfData := &pageFileData{
name: pageFileName,
used: enumPageFileInfo.totalInUse,
total: enumPageFileInfo.totalSize,
}

*pageFiles = append(*pageFiles, pfData)

// return true to continue enumerating page files
ret := true
return &ret
}

0 comments on commit 3d83031

Please sign in to comment.