From 4e99ee83a70e761ccb24694ac81981f72ead0996 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 13:12:28 +0200 Subject: [PATCH] Use common host parser in vsphere module (#26904) Use the common host parser builder to parse hosts defined in vsphere module configuration. Since #21022, sanitized URIs included in modules host data are used as `service.address`. vsphere did a custom parsing that didn't fill the sanitized URI and then `service.address` was not filled. (cherry picked from commit 68e990993fc5b6ce9e360befc2bef53713aaf8c0) --- CHANGELOG.next.asciidoc | 1 + .../module/vsphere/datastore/_meta/data.json | 10 ++-- .../module/vsphere/datastore/datastore.go | 32 +++------- .../module/vsphere/host/_meta/data.json | 15 +++-- metricbeat/module/vsphere/host/host.go | 32 +++------- metricbeat/module/vsphere/metricset.go | 59 +++++++++++++++++++ metricbeat/module/vsphere/test_vsphere.py | 6 ++ .../vsphere/virtualmachine/_meta/data.json | 5 +- .../vsphere/virtualmachine/virtualmachine.go | 33 ++++------- 9 files changed, 107 insertions(+), 86 deletions(-) create mode 100644 metricbeat/module/vsphere/metricset.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 469ae0e85f6..11f018ea1bb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220] - Added Airflow lightweight module {pull}26220[26220] - Add state_job metricset to Kubernetes module{pull}26479[26479] +- Recover service.address field in vsphere module {issue}26902[26902] {pull}26904[26904] *Packetbeat* diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json index fb290513a92..944ecb1189f 100644 --- a/metricbeat/module/vsphere/datastore/_meta/data.json +++ b/metricbeat/module/vsphere/datastore/_meta/data.json @@ -10,21 +10,21 @@ "period": 10000 }, "service": { - "address": "http://127.0.0.1:51027/sdk", + "address": "127.0.0.1:33365", "type": "vsphere" }, "vsphere": { "datastore": { "capacity": { "free": { - "bytes": 267958476800 + "bytes": 37120094208 }, "total": { - "bytes": 484736266240 + "bytes": 74686664704 }, "used": { - "bytes": 216777789440, - "pct": 0.44720769733509097 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "local", diff --git a/metricbeat/module/vsphere/datastore/datastore.go b/metricbeat/module/vsphere/datastore/datastore.go index f794384d453..f37171c65a3 100644 --- a/metricbeat/module/vsphere/datastore/datastore.go +++ b/metricbeat/module/vsphere/datastore/datastore.go @@ -19,12 +19,12 @@ package datastore import ( "context" - "net/url" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/vmware/govmomi" "github.com/vmware/govmomi/view" @@ -33,41 +33,23 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "datastore", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet } -// New create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - }{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - - u, err := url.Parse(base.HostData().URI) + ms, err := vsphere.NewMetricSet(base) if err != nil { return nil, err } - - u.User = url.UserPassword(config.Username, config.Password) - - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, - }, nil + return &MetricSet{ms}, nil } // Fetch methods implements the data gathering and data conversion to the right diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index c1948cdc304..05443939f10 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -1,14 +1,17 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "vsphere.host", + "duration": 115000, + "module": "vsphere" }, "metricset": { - "host": "http://127.0.0.1:43843/sdk", - "module": "vsphere", "name": "host", - "rtt": 115 + "period": 10000 + }, + "service": { + "address": "127.0.0.1:38517", + "type": "vsphere" }, "vsphere": { "host": { diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index e8bc5d090bb..6f7d19b5d57 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -20,13 +20,13 @@ package host import ( "context" "fmt" - "net/url" "strings" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/vmware/govmomi" "github.com/vmware/govmomi/property" @@ -38,41 +38,23 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "host", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet } -// New create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - }{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - - u, err := url.Parse(base.HostData().URI) + ms, err := vsphere.NewMetricSet(base) if err != nil { return nil, err } - - u.User = url.UserPassword(config.Username, config.Password) - - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, - }, nil + return &MetricSet{ms}, nil } // Fetch methods implements the data gathering and data conversion to the right diff --git a/metricbeat/module/vsphere/metricset.go b/metricbeat/module/vsphere/metricset.go new file mode 100644 index 00000000000..c7089b52ac7 --- /dev/null +++ b/metricbeat/module/vsphere/metricset.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package vsphere + +import ( + "net/url" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: "https", + DefaultPath: "/sdk", +}.Build() + +// MetricSet type defines all fields of the MetricSet. +type MetricSet struct { + mb.BaseMetricSet + Insecure bool + HostURL *url.URL +} + +// NewMetricSet creates a new instance of the MetricSet. +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + config := struct { + Insecure bool `config:"insecure"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + u, err := url.Parse(base.HostData().URI) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + HostURL: u, + Insecure: config.Insecure, + }, nil +} diff --git a/metricbeat/module/vsphere/test_vsphere.py b/metricbeat/module/vsphere/test_vsphere.py index c0c2b0db4bb..edc06a16c9f 100644 --- a/metricbeat/module/vsphere/test_vsphere.py +++ b/metricbeat/module/vsphere/test_vsphere.py @@ -42,6 +42,8 @@ def test_datastore(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @@ -71,6 +73,8 @@ def test_host(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @@ -100,4 +104,6 @@ def test_virtualmachine(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/data.json b/metricbeat/module/vsphere/virtualmachine/_meta/data.json index 4dfa302758c..2dea5c91363 100644 --- a/metricbeat/module/vsphere/virtualmachine/_meta/data.json +++ b/metricbeat/module/vsphere/virtualmachine/_meta/data.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "http://127.0.0.1:37231/sdk", + "address": "127.0.0.1:39149", "type": "vsphere" }, "vsphere": { @@ -43,9 +43,6 @@ } }, "name": "ha-host_VM0", - "network_names": [ - "VM Network" - ], "os": "otherGuest" } } diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go index 938bf91c97b..13af0168765 100644 --- a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go +++ b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go @@ -20,11 +20,11 @@ package virtualmachine import ( "context" "fmt" - "net/url" "strings" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/pkg/errors" "github.com/vmware/govmomi" @@ -38,25 +38,26 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "virtualmachine", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet GetCustomFields bool } -// New create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := vsphere.NewMetricSet(base) + if err != nil { + return nil, err + } + config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - GetCustomFields bool `config:"get_custom_fields"` + GetCustomFields bool `config:"get_custom_fields"` }{ GetCustomFields: false, } @@ -64,18 +65,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - - u, err := url.Parse(base.HostData().URI) - if err != nil { - return nil, err - } - - u.User = url.UserPassword(config.Username, config.Password) - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, + MetricSet: ms, GetCustomFields: config.GetCustomFields, }, nil }