Skip to content

Commit

Permalink
Add host inventory metrics to azure compute_vm metricset (elastic#20641)
Browse files Browse the repository at this point in the history 
* Add host inventory metrics to azure compute_vm metricset
  • Loading branch information
@kaiyan-sheng @melchiormoulin
kaiyan-sheng authored and melchiormoulin committed Oct 14, 2020
1 parent e54a31e commit bdcd991
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 97 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077]
- Add host inventory metrics into aws ec2 metricset. {pull}20171[20171]
- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
- Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]

*Packetbeat*
Expand Down
221 changes: 124 additions & 97 deletions x-pack/metricbeat/module/azure/compute_vm/_meta/data.json
View file
Original file line number Diff line number Diff line change
@@ -1,131 +1,158 @@
{
"@timestamp":"2020-08-05T10:36:00.000Z",
"cloud":{
"provider":"azure",
"region":"westeurope",
"instance":{
"id":"/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak",
"name":"obstestmemleak"
},
"machine":{
"type":"Standard_B2ms"
}
},
"event":{
"dataset":"azure.compute_vm",
"module":"azure",
"duration":5364973100
},
"metricset":{
"name":"compute_vm",
"period":300000
},
"azure":{
"timegrain":"PT5M",
"resource":{
"type":"Microsoft.Compute/virtualMachines",
"group":"obs-infrastructure",
"tags":{
"vmtest":"value1, value 2",
"vmtest1":"value3"
}
},
"subscription_id":"fd675b6f-b5e5-426e-ac45-d1f876d0ffa6",
"namespace":"Microsoft.Compute/virtualMachines",
"compute_vm":{
"outbound_flows":{
"avg":291.2
"@timestamp": "2017-10-12T08:05:34.853Z",
"azure": {
"compute_vm": {
"cpu_credits_consumed": {
"avg": 0.01
},
"cpu_credits_remaining": {
"avg": 288
},
"disk_read_bytes": {
"total": 4019543.24
},
"outbound_flows_maximum_creation_rate":{
"avg":6
"disk_read_operations_per_sec": {
"avg": 0.0605
},
"inbound_flows_maximum_creation_rate":{
"avg":6
"disk_write_bytes": {
"total": 16739967.27
},
"os_per_disk_read_bytes_per_sec":{
"avg":341.35875
"disk_write_operations_per_sec": {
"avg": 1.9525
},
"per_disk_read_bytes_per_sec":{
"avg":0
"inbound_flows": {
"avg": 31
},
"os_disk_queue_depth":{
"avg":0
"inbound_flows_maximum_creation_rate": {
"avg": 0.8
},
"os_per_disk_qd":{
"avg":0
"network_in": {
"total": 1501534
},
"os_disk_read_bytes_per_sec":{
"avg":341.35875
"network_in_total": {
"total": 1806936
},
"network_in":{
"total":3933481.0
"network_out": {
"total": 1647640
},
"network_out":{
"total":7169752.0
"network_out_total": {
"total": 3633130
},
"os_disk_write_operations_per_sec":{
"avg":4.994285714285715
"os_disk_queue_depth": {
"avg": 0
},
"percentage_cpu":{
"avg":8.780625
"os_disk_read_bytes_per_sec": {
"avg": 13398.475
},
"cpu_credits_remaining":{
"avg":864
"os_disk_read_operations_per_sec": {
"avg": 0.121
},
"network_out_total":{
"total":9249694.0
"os_disk_write_bytes_per_sec": {
"avg": 55799.886
},
"disk_read_operations_per_sec":{
"avg":0.022857142857142857
"os_disk_write_operations_per_sec": {
"avg": 3.905
},
"disk_write_bytes":{
"total":1.759878456E7
"os_per_disk_qd": {
"avg": 0
},
"os_disk_write_bytes_per_sec":{
"avg":73050.70375
"os_per_disk_read_bytes_per_sec": {
"avg": 13398.475
},
"cpu_credits_consumed":{
"avg":0.175
"os_per_disk_read_operations_per_sec": {
"avg": 0.121
},
"per_disk_qd":{
"avg":0
"os_per_disk_write_bytes_per_sec": {
"avg": 55799.886
},
"disk_write_operations_per_sec":{
"avg":2.5221428571428572
"os_per_disk_write_operations_per_sec": {
"avg": 3.905
},
"os_per_disk_write_operations_per_sec":{
"avg":4.994285714285715
"outbound_flows": {
"avg": 31
},
"per_disk_write_operations_per_sec":{
"avg":0.049999999999999996
"outbound_flows_maximum_creation_rate": {
"avg": 0.8
},
"inbound_flows":{
"avg":291.2
"per_disk_qd": {
"avg": 0
},
"os_disk_read_operations_per_sec":{
"avg":0.045714285714285714
"per_disk_read_bytes_per_sec": {
"avg": 0
},
"disk_read_bytes":{
"total":81926.52
"per_disk_read_operations_per_sec": {
"avg": 0
},
"network_in_total":{
"total":4920267.0
"per_disk_write_bytes_per_sec": {
"avg": 0
},
"per_disk_read_operations_per_sec":{
"avg":0
"per_disk_write_operations_per_sec": {
"avg": 0
},
"per_disk_write_bytes_per_sec":{
"avg":277.56
"percentage_cpu": {
"avg": 1.585
},
"os_per_disk_write_bytes_per_sec":{
"avg":73050.70375
"premium_os_disk_cache_read_hit": {
"avg": 100
},
"premium_os_disk_cache_read_miss": {
"avg": 0
}
},
"namespace": "Microsoft.Compute/virtualMachines",
"resource": {
"group": "obs-test",
"type": "Microsoft.Compute/virtualMachines"
},
"subscription_id": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba",
"timegrain": "PT5M"
},
"cloud": {
"instance": {
"id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test",
"name": "perfmon-test"
},
"machine": {
"type": "Standard_B1ms"
},
"provider": "azure",
"region": "westeurope"
},
"event": {
"dataset": "azure.compute_vm",
"duration": 115000,
"module": "azure"
},
"host": {
"cpu": {
"pct": 0.01585
},
"disk": {
"read": {
"bytes": 4019543.24
},
"os_per_disk_read_operations_per_sec":{
"avg":0.045714285714285714
"write": {
"bytes": 16739967.27
}
},
"id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test",
"name": "perfmon-test",
"network": {
"in": {
"bytes": 1806936,
"packets": 1501534
},
"out": {
"bytes": 3633130,
"packets": 1647640
}
}
},
"service":{
"type":"azure"
"metricset": {
"name": "compute_vm",
"period": 10000
},
"service": {
"type": "azure"
}
}
}
31 changes: 31 additions & 0 deletions x-pack/metricbeat/module/azure/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,10 +173,13 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue)
event.RootFields.Put("cloud.provider", "azure")
event.RootFields.Put("cloud.region", metric.Resource.Location)
event.RootFields.Put("cloud.instance.name", metric.Resource.Name)
event.RootFields.Put("host.name", metric.Resource.Name)
if metric.Resource.SubId != "" {
event.RootFields.Put("cloud.instance.id", metric.Resource.SubId)
event.RootFields.Put("host.id", metric.Resource.SubId)
} else {
event.RootFields.Put("cloud.instance.id", metric.Resource.Id)
event.RootFields.Put("host.id", metric.Resource.Id)
}
if metric.Resource.Size != "" {
event.RootFields.Put("cloud.machine.type", metric.Resource.Size)
Expand All @@ -200,6 +203,7 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue)
metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "count"), *value.count)
}
}
event = addHostFields(event, metricList)
return event, metricList
}

Expand Down Expand Up @@ -229,3 +233,30 @@ func returnAllDimensions(dimensions []Dimension) (bool, []Dimension) {
}
return true, dims
}

func addHostFields(event mb.Event, metricList common.MapStr) mb.Event {
hostFieldTable := map[string]string{
"percentage_cpu.avg": "host.cpu.pct",
"network_in_total.total": "host.network.in.bytes",
"network_in.total": "host.network.in.packets",
"network_out_total.total": "host.network.out.bytes",
"network_out.total": "host.network.out.packets",
"disk_read_bytes.total": "host.disk.read.bytes",
"disk_write_bytes.total": "host.disk.write.bytes",
}

for metricName, hostName := range hostFieldTable {
metricValue, err := metricList.GetValue(metricName)
if err != nil {
continue
}

if value, ok := metricValue.(float64); ok {
if metricName == "percentage_cpu.avg" {
value = value / 100
}
event.RootFields.Put(hostName, value)
}
}
return event
}

0 comments on commit bdcd991

Please sign in to comment.