Move pdh query to shared location in order for new modules/metricsets to reuse

CHANGELOG.next.asciidoc

@@ -82,6 +82,100 @@ TLS or Beats that accept connections over TLS and validate client certificates.
 
 *Metricbeat*
 
+- Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053]
+- Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055]
+- Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055]
+- Add `coredns` metricbeat module. {pull}10585[10585]
+- Add SSL support for Metricbeat HTTP server. {pull}11482[11482] {issue}11457[11457]
+- The `elasticsearch.index` metricset (with `xpack.enabled: true`) now collects `refresh.external_total_time_in_millis` fields from Elasticsearch. {pull}11616[11616]
+- Allow module configurations to have variants {pull}9118[9118]
+- Add `timeseries.instance` field calculation. {pull}10293[10293]
+- Added new disk states and raid level to the system/raid metricset. {pull}11613[11613]
+- Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877]
+- Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878]
+- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734]
+- Add `regions` in aws module config to specify target regions for querying cloudwatch metrics. {issue}11932[11932] {pull}11956[11956]
+- Keep `etcd` followers members from reporting `leader` metricset events {pull}12004[12004]
+- Add overview dashboard to Consul module {pull}10665[10665]
+- New fields were added in the mysql/status metricset. {pull}12227[12227]
+- Add Kubernetes metricset `proxy`. {pull}12312[12312]
+- Add Kubernetes proxy dashboard to Kubernetes module {pull}12734[12734]
+- Always report Pod UID in the `pod` metricset. {pull}12345[12345]
+- Add Vsphere Virtual Machine operating system to `os` field in Vsphere virtualmachine module. {pull}12391[12391]
+- Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386]
+- Add CockroachDB module. {pull}12467[12467]
+- Add support for metricbeat modules based on existing modules (a.k.a. light modules) {issue}12270[12270] {pull}12465[12465]
+- Add a system/entropy metricset {pull}12450[12450]
+- Add kubernetes metricset `controllermanager` {pull}12409[12409]
+- Add Kubernetes controller manager dashboard to Kubernetes module {pull}12744[12744]
+- Allow redis URL format in redis hosts config. {pull}12408[12408]
+- Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372]
+- Add metrics to kubernetes apiserver metricset. {pull}12922[12922]
+- Add kubernetes metricset `scheduler` {pull}12521[12521]
+- Add Kubernetes scheduler dashboard to Kubernetes module {pull}12749[12749]
+- Add `beat` module. {pull}12181[12181] {pull}12615[12615]
+- Collect tags for cloudwatch metricset in aws module. {issue}[12263]12263 {pull}12480[12480]
+- Add AWS RDS metricset. {pull}11620[11620] {issue}10054[10054]
+- Add Oracle Module {pull}11890[11890]
+- Add Oracle Tablespaces Dashboard {pull}12736[12736]
+- Collect client provided name for rabbitmq connection. {issue}12851[12851] {pull}12852[12852]
+- Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708]
+- Add statistic option into cloudwatch metricset. {issue}12370[12370] {pull}12840[12840]
+- Add support for kubernetes cronjobs {pull}13001[13001]
+- Add cgroup memory stats to docker/memory metricset {pull}12916[12916]
+- Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701]
+- Add AWS ebs metricset. {pull}13167[13167] {issue}11699[11699]
+- Add `metricset.period` field with the configured fetching period. {pull}13242[13242] {issue}12616[12616]
+- Add rate metrics for ec2 metricset. {pull}13203[13203]
+- Add refresh list of perf counters at every fetch {issue}13091[13091]
+- Add Performance metricset to Oracle module {pull}12547[12547]
+- Add proc/vmstat data to the system/memory metricset on linux {pull}13322[13322]
+- Use DefaultMetaGeneratorConfig in MetadataEnrichers to initialize configurations {pull}13414[13414]
+- Add module for statsd. {pull}13109[13109]
+- Add support for NATS version 2. {pull}13601[13601]
+- Add `docker.cpu.*.norm.pct` metrics for `cpu` metricset of Docker Metricbeat module. {pull}13695[13695]
+- Add `instance` label by default when using Prometheus collector. {pull}13737[13737]
+- Add azure module. {pull}13196[13196] {pull}13859[13859] {pull}13988[13988]
+- Add Apache Tomcat module {pull}13491[13491]
+- Add ECS `container.id` and `container.runtime` to kubernetes `state_container` metricset. {pull}13884[13884]
+- Add `job` label by default when using Prometheus collector. {pull}13878[13878]
+- Add `state_resourcequota` metricset for Kubernetes module. {pull}13693[13693]
+- Add tags filter in ec2 metricset. {pull}13872[13872] {issue}13145[13145]
+- Add cloud.account.id and cloud.account.name into events from aws module. {issue}13551[13551] {pull}13558[13558]
+- Add `metrics_path` as known hint for autodiscovery {pull}13996[13996]
+- Leverage KUBECONFIG when creating k8s client. {pull}13916[13916]
+- Add ability to filter by tags for cloudwatch metricset. {pull}13758[13758] {issue}13145[13145]
+- Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059]
+- Add Oracle overview dashboard {pull}14021[14021]
+- Release CoreDNS module as GA. {pull}14308[14308]
+- Release CouchDB module as GA. {pull}14300[14300]
+- Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
+- Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539]
+- Release aws ebs metricset as GA. {pull}14312[14312] {issue}14060[14060]
+- Add `connection.state` field for RabbitMQ module. {pull}13981[13981]
+- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
+- Add Kafka JMX metricsets. {pull}14330[14330]
+- Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
+- Release kubernetes modules `controllermanager`, `scheduler`, `proxy`, `state_cronjob` and `state_resourcequota` as GA. {pull}14584[14584]
+- Add module for ActiveMQ. {pull}14580[14580]
+- Enable script processor. {pull}14711[14711]
+- Enable wildcard for cloudwatch metricset namespace. {pull}14971[14971] {issue}14965[14965]
+- Add `kube-state-metrics` `state_service` metrics for kubernetes module. {pull}14794[14794]
+- Add `kube-state-metrics` `state_persistentvolume` metrics for kubernetes module. {pull}14859[14859]
+- Add `kube-state-metrics` `state_persistentvolumeclaim` metrics for kubernetes module. {pull}15066[15066]
+- Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935]
+- Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934]
+- Add AWS SNS metricset. {pull}14946[14946]
+- Add overview dashboard for AWS SNS module {pull}14977[14977]
+- Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100]
+- Add a `system/service` metricset for systemd data. {pull}14206[14206]
+- Expand data for the `system/memory` metricset {pull}15492[15492]
+- Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
+- Add cost warnings for the azure module. {pull}15356[15356]
+- Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503]
+- Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
+- Release elb module as GA. {pull}15485[15485]
+- Add a `system/network_summary` metricset {pull}15196[15196]
 
 *Packetbeat*
 

metricbeat/module/windows/perfmon/defs_pdh_windows.go → metricbeat/helper/windows/pdh/defs_pdh_windows.go

@@ -20,7 +20,7 @@
 
 // +build ignore
 
-package perfmon
+package pdh
 
 /*
 #include <pdh.h>

metricbeat/module/windows/doc.go → metricbeat/helper/windows/pdh/doc.go

@@ -15,7 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-/*
-Package windows is a Metricbeat module that contains MetricSets.
-*/
-package windows
+package pdh
+
+//go:generate go run mkpdh_defs.go
+//go:generate go run ../run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch amd64 -output defs_pdh_windows_amd64.go
+//go:generate go run ../run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch 386 -output defs_pdh_windows_386.go
+//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zpdh_windows.go pdh_windows.go
+//go:generate gofmt -w defs_pdh_windows_amd64.go defs_pdh_windows_386.go zpdh_windows.go

metricbeat/module/windows/perfmon/mkpdh_defs.go → metricbeat/helper/windows/pdh/mkpdh_defs.go

@@ -48,7 +48,7 @@ const fileTemplate = `
 
 // +build ignore
 
-package perfmon
+package pdh
 
 /*
 #include <pdh.h>

metricbeat/module/windows/perfmon/pdh_query_windows.go → metricbeat/helper/windows/pdh/pdh_query_windows.go

@@ -17,7 +17,7 @@
 
 // +build windows
 
-package perfmon
+package pdh
 
 import (
 	"regexp"
@@ -43,8 +43,8 @@ type Counter struct {
 
 // Query contains the pdh.
 type Query struct {
-	handle   PdhQueryHandle
-	counters map[string]*Counter
+	Handle   PdhQueryHandle
+	Counters map[string]*Counter
 }
 
 // CounterValue contains the performance counter values.
@@ -60,42 +60,42 @@ func (q *Query) Open() error {
 	if err != nil {
 		return err
 	}
-	q.handle = h
-	q.counters = make(map[string]*Counter)
+	q.Handle = h
+	q.Counters = make(map[string]*Counter)
 	return nil
 }
 
 // AddEnglishCounter adds the specified counter to the query.
 func (q *Query) AddEnglishCounter(counterPath string) (PdhCounterHandle, error) {
-	h, err := PdhAddEnglishCounter(q.handle, counterPath, 0)
+	h, err := PdhAddEnglishCounter(q.Handle, counterPath, 0)
 	return h, err
 }
 
 // AddCounter adds the specified counter to the query.
-func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard bool) error {
-	if _, found := q.counters[counterPath]; found {
+func (q *Query) AddCounter(counterPath string, instance string, format string, wildcard bool) error {
+	if _, found := q.Counters[counterPath]; found {
 		return nil
 	}
 	var err error
 	var instanceName string
 	// Extract the instance name from the counterPath.
-	if counter.InstanceName == "" || wildcard {
+	if instance == "" || wildcard {
 		instanceName, err = matchInstanceName(counterPath)
 		if err != nil {
 			return err
 		}
 	} else {
-		instanceName = counter.InstanceName
+		instanceName = instance
 	}
-	h, err := PdhAddCounter(q.handle, counterPath, 0)
+	h, err := PdhAddCounter(q.Handle, counterPath, 0)
 	if err != nil {
 		return err
 	}
 
-	q.counters[counterPath] = &Counter{
+	q.Counters[counterPath] = &Counter{
 		handle:       h,
 		instanceName: instanceName,
-		format:       getPDHFormat(counter.Format),
+		format:       getPDHFormat(format),
 	}
 	return nil
 }
@@ -134,7 +134,7 @@ func (q *Query) RemoveUnusedCounters(counters []string) error {
 		}
 	}
 	unused := make(map[string]*Counter)
-	for counterPath, counter := range q.counters {
+	for counterPath, counter := range q.Counters {
 		if !matchCounter(counterPath, counters) {
 			unused[counterPath] = counter
 		}
@@ -147,7 +147,7 @@ func (q *Query) RemoveUnusedCounters(counters []string) error {
 		if err != nil {
 			return err
 		}
-		delete(q.counters, counterPath)
+		delete(q.Counters, counterPath)
 	}
 	return nil
 }
@@ -163,16 +163,16 @@ func matchCounter(counterPath string, counterList []string) bool {
 
 // CollectData collects the value for all counters in the query.
 func (q *Query) CollectData() error {
-	return PdhCollectQueryData(q.handle)
+	return PdhCollectQueryData(q.Handle)
 }
 
 // GetFormattedCounterValues returns an array of formatted values for a query.
 func (q *Query) GetFormattedCounterValues() (map[string][]CounterValue, error) {
-	if q.counters == nil || len(q.counters) == 0 {
+	if q.Counters == nil || len(q.Counters) == 0 {
 		return nil, errors.New("no counter list found")
 	}
-	rtn := make(map[string][]CounterValue, len(q.counters))
-	for path, counter := range q.counters {
+	rtn := make(map[string][]CounterValue, len(q.Counters))
+	for path, counter := range q.Counters {
 		rtn[path] = append(rtn[path], getCounterValue(counter))
 	}
 	return rtn, nil
@@ -206,7 +206,7 @@ func (q *Query) ExpandWildCardPath(wildCardPath string) ([]string, error) {
 
 // Close closes the query and all of its counters.
 func (q *Query) Close() error {
-	return PdhCloseQuery(q.handle)
+	return PdhCloseQuery(q.Handle)
 }
 
 // matchInstanceName will check first for instance and then for any objects names.

metricbeat/module/windows/perfmon/pdh_query_windows_test.go → metricbeat/helper/windows/pdh/pdh_query_windows_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package perfmon
+package pdh
 
 import (
 	"syscall"
@@ -38,8 +38,7 @@ func TestAddCounterInvalidArgWhenQueryClosed(t *testing.T) {
 	queryPath, err := q.GetCounterPaths(validQuery)
 	// if windows os language is ENG then err will be nil, else the GetCounterPaths will execute the AddCounter
 	if assert.NoError(t, err) {
-		counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"}
-		err = q.AddCounter(queryPath[0], counter, false)
+		err = q.AddCounter(queryPath[0], "TestInstanceName", "float", false)
 		assert.Error(t, err, PDH_INVALID_HANDLE)
 	} else {
 		assert.Error(t, err, PDH_INVALID_ARGUMENT)
@@ -70,12 +69,11 @@ func TestSuccessfulQuery(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer q.Close()
-	counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"}
 	queryPath, err := q.GetCounterPaths(validQuery)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = q.AddCounter(queryPath[0], counter, false)
+	err = q.AddCounter(queryPath[0], "TestInstanceName", "floar", false)
 	if err != nil {
 		t.Fatal(err)
 	}

metricbeat/module/windows/perfmon/pdh_windows.go → metricbeat/helper/windows/pdh/pdh_windows.go

@@ -17,7 +17,7 @@
 
 // +build windows
 
-package perfmon
+package pdh
 
 import (
 	"strconv"

metricbeat/module/windows/perfmon/pdh_windows_test.go → metricbeat/helper/windows/pdh/pdh_windows_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package perfmon
+package pdh
 
 import (
 	"syscall"

metricbeat/module/windows/perfmon/doc.go

@@ -15,12 +15,4 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// Package perfmon implements a Metricbeat metricset for reading Windows
-// performance counters.
 package perfmon
-
-//go:generate go run mkpdh_defs.go
-//go:generate go run ../run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch amd64 -output defs_pdh_windows_amd64.go
-//go:generate go run ../run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch 386 -output defs_pdh_windows_386.go
-//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zpdh_windows.go pdh_windows.go
-//go:generate gofmt -w defs_pdh_windows_amd64.go defs_pdh_windows_386.go zpdh_windows.go

metricbeat/module/windows/perfmon/perfmon.go

@@ -90,7 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // Fetch fetches events and reports them upstream
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	// if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before
-	if len(m.reader.query.counters) == 0 {
+	if len(m.reader.query.Counters) == 0 {
 		return errors.New("no counters to read")
 	}