From 909d78995af2610b01f0d5b3e696659d79deedce Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 11:44:02 -0700 Subject: [PATCH] Make logstash module code consistent with other Elastic stack modules (#8497) This PR makes the Metricbeat `logstash` module code consistent with other Elastic stack modules by: * using `FetchContent()` instead of `FetchJSON()` to fetch data from LS APIs before passing it to the event mapping function. * adding unit tests. Further consistency around event reporting and error handling will be achieved in #8308. --- .../logstash/node/_meta/test/node.641.json | 41 ++++++ .../logstash/node/_meta/test/node.650.json | 41 ++++++ .../logstash/node/_meta/test/node.700.json | 41 ++++++ metricbeat/module/logstash/node/data.go | 12 +- metricbeat/module/logstash/node/data_test.go | 44 ++++++ metricbeat/module/logstash/node/node.go | 4 +- .../node_stats/_meta/test/node_stats.641.json | 129 ++++++++++++++++++ .../node_stats/_meta/test/node_stats.650.json | 129 ++++++++++++++++++ .../node_stats/_meta/test/node_stats.700.json | 129 ++++++++++++++++++ metricbeat/module/logstash/node_stats/data.go | 11 +- .../module/logstash/node_stats/data_test.go | 44 ++++++ .../module/logstash/node_stats/node_stats.go | 4 +- 12 files changed, 621 insertions(+), 8 deletions(-) create mode 100644 metricbeat/module/logstash/node/_meta/test/node.641.json create mode 100644 metricbeat/module/logstash/node/_meta/test/node.650.json create mode 100644 metricbeat/module/logstash/node/_meta/test/node.700.json create mode 100644 metricbeat/module/logstash/node/data_test.go create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.641.json create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.650.json create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.700.json create mode 100644 metricbeat/module/logstash/node_stats/data_test.go diff --git a/metricbeat/module/logstash/node/_meta/test/node.641.json b/metricbeat/module/logstash/node/_meta/test/node.641.json new file mode 100644 index 00000000000..5e86e3cb797 --- /dev/null +++ b/metricbeat/module/logstash/node/_meta/test/node.641.json @@ -0,0 +1,41 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "6.4.1", + "http_address": "127.0.0.1:9600", + "id": "01cf780e-b576-4e59-9ee0-6b49d73e1238", + "name": "Shaunaks-MBP-2", + "pipelines": { + "main": { + "workers": 8, + "batch_size": 125, + "batch_delay": 50, + "config_reload_automatic": false, + "config_reload_interval": 3000000000, + "dead_letter_queue_enabled": false + } + }, + "os": { + "name": "Mac OS X", + "arch": "x86_64", + "version": "10.13.6", + "available_processors": 8 + }, + "jvm": { + "pid": 92230, + "version": "1.8.0_171", + "vm_version": "1.8.0_171", + "vm_vendor": "Oracle Corporation", + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "start_time_in_millis": 1538404190630, + "mem": { + "heap_init_in_bytes": 1073741824, + "heap_max_in_bytes": 1037959168, + "non_heap_init_in_bytes": 2555904, + "non_heap_max_in_bytes": 0 + }, + "gc_collectors": [ + "ParNew", + "ConcurrentMarkSweep" + ] + } +} diff --git a/metricbeat/module/logstash/node/_meta/test/node.650.json b/metricbeat/module/logstash/node/_meta/test/node.650.json new file mode 100644 index 00000000000..9f17a9c7e92 --- /dev/null +++ b/metricbeat/module/logstash/node/_meta/test/node.650.json @@ -0,0 +1,41 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "6.5.0", + "http_address": "127.0.0.1:9600", + "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", + "name": "Shaunaks-MBP-2", + "pipelines": { + "main": { + "workers": 8, + "batch_size": 125, + "batch_delay": 50, + "config_reload_automatic": false, + "config_reload_interval": 3000000000, + "dead_letter_queue_enabled": false + } + }, + "os": { + "name": "Mac OS X", + "arch": "x86_64", + "version": "10.13.6", + "available_processors": 8 + }, + "jvm": { + "pid": 1559, + "version": "1.8.0_171", + "vm_version": "1.8.0_171", + "vm_vendor": "Oracle Corporation", + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "start_time_in_millis": 1538406837613, + "mem": { + "heap_init_in_bytes": 1073741824, + "heap_max_in_bytes": 1037959168, + "non_heap_init_in_bytes": 2555904, + "non_heap_max_in_bytes": 0 + }, + "gc_collectors": [ + "ParNew", + "ConcurrentMarkSweep" + ] + } +} diff --git a/metricbeat/module/logstash/node/_meta/test/node.700.json b/metricbeat/module/logstash/node/_meta/test/node.700.json new file mode 100644 index 00000000000..d084786ac00 --- /dev/null +++ b/metricbeat/module/logstash/node/_meta/test/node.700.json @@ -0,0 +1,41 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "7.0.0-alpha1", + "http_address": "127.0.0.1:9600", + "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", + "name": "Shaunaks-MBP-2", + "pipelines": { + "main": { + "workers": 8, + "batch_size": 125, + "batch_delay": 50, + "config_reload_automatic": false, + "config_reload_interval": 3000000000, + "dead_letter_queue_enabled": false + } + }, + "os": { + "name": "Mac OS X", + "arch": "x86_64", + "version": "10.13.6", + "available_processors": 8 + }, + "jvm": { + "pid": 83353, + "version": "1.8.0_171", + "vm_version": "1.8.0_171", + "vm_vendor": "Oracle Corporation", + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "start_time_in_millis": 1538217473740, + "mem": { + "heap_init_in_bytes": 1073741824, + "heap_max_in_bytes": 1037959168, + "non_heap_init_in_bytes": 2555904, + "non_heap_max_in_bytes": 0 + }, + "gc_collectors": [ + "ParNew", + "ConcurrentMarkSweep" + ] + } +} diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index eb1b5ba61f5..24645edaba2 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -18,6 +18,8 @@ package node import ( + "encoding/json" + "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" @@ -34,6 +36,12 @@ var ( } ) -func eventMapping(node map[string]interface{}) (common.MapStr, error) { - return schema.Apply(node) +func eventMapping(content []byte) (common.MapStr, error) { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return nil, err + } + + return schema.Apply(data) } diff --git a/metricbeat/module/logstash/node/data_test.go b/metricbeat/module/logstash/node/data_test.go new file mode 100644 index 00000000000..104fec15aa0 --- /dev/null +++ b/metricbeat/module/logstash/node/data_test.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package node + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/node.*.json") + assert.NoError(t, err) + + for _, f := range files { + content, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + event, err := eventMapping(content) + + assert.NoError(t, err, f) + assert.True(t, len(event) >= 1, f) + } +} diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index 15a316f8b9f..35601ed84b7 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -65,11 +65,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch() (common.MapStr, error) { - data, err := m.http.FetchJSON() + content, err := m.http.FetchContent() if err != nil { return nil, err } - event, _ := eventMapping(data) + event, _ := eventMapping(content) return event, nil } diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.641.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.641.json new file mode 100644 index 00000000000..6bcf88f1940 --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.641.json @@ -0,0 +1,129 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "6.4.1", + "http_address": "127.0.0.1:9600", + "id": "01cf780e-b576-4e59-9ee0-6b49d73e1238", + "name": "Shaunaks-MBP-2", + "jvm": { + "threads": { + "count": 30, + "peak_count": 32 + }, + "mem": { + "heap_used_percent": 32, + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 341759888, + "non_heap_used_in_bytes": 121819632, + "non_heap_committed_in_bytes": 140050432, + "pools": { + "survivor": { + "peak_used_in_bytes": 35782656, + "used_in_bytes": 21321184, + "peak_max_in_bytes": 35782656, + "max_in_bytes": 35782656, + "committed_in_bytes": 35782656 + }, + "old": { + "peak_used_in_bytes": 103570264, + "used_in_bytes": 91787528, + "peak_max_in_bytes": 715849728, + "max_in_bytes": 715849728, + "committed_in_bytes": 715849728 + }, + "young": { + "peak_used_in_bytes": 286326784, + "used_in_bytes": 228651176, + "peak_max_in_bytes": 286326784, + "max_in_bytes": 286326784, + "committed_in_bytes": 286326784 + } + } + }, + "gc": { + "collectors": { + "old": { + "collection_time_in_millis": 184, + "collection_count": 2 + }, + "young": { + "collection_time_in_millis": 413, + "collection_count": 8 + } + } + }, + "uptime_in_millis": 70624 + }, + "process": { + "open_file_descriptors": 95, + "peak_open_file_descriptors": 96, + "max_file_descriptors": 10240, + "mem": { + "total_virtual_in_bytes": 7176814592 + }, + "cpu": { + "total_in_millis": 100279, + "percent": 0, + "load_average": { + "1m": 5.107421875 + } + } + }, + "events": { + "in": 0, + "filtered": 0, + "out": 0, + "duration_in_millis": 0, + "queue_push_duration_in_millis": 0 + }, + "pipelines": { + "main": { + "events": { + "duration_in_millis": 0, + "in": 0, + "out": 0, + "filtered": 0, + "queue_push_duration_in_millis": 0 + }, + "plugins": { + "inputs": [ + { + "id": "fc1bd2827edc6b55a3caf4af83c5ac8c5c32b84cb54453bdac8c0d1dcb37ee46", + "events": { + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "name": "stdin" + } + ], + "filters": [], + "outputs": [ + { + "id": "47e3be4432eda99c8d50ea91fdec98aa9c3d5689589ddd3cfdbd62b43746d841", + "events": { + "duration_in_millis": 0, + "in": 0, + "out": 0 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "successes": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "failures": 0 + }, + "queue": { + "type": "memory" + } + } + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "os": {} +} diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.650.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.650.json new file mode 100644 index 00000000000..cdd290d405a --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.650.json @@ -0,0 +1,129 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "6.5.0", + "http_address": "127.0.0.1:9600", + "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", + "name": "Shaunaks-MBP-2", + "jvm": { + "threads": { + "count": 30, + "peak_count": 32 + }, + "mem": { + "heap_used_percent": 34, + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 362846688, + "non_heap_used_in_bytes": 126413504, + "non_heap_committed_in_bytes": 144498688, + "pools": { + "survivor": { + "peak_used_in_bytes": 35782656, + "used_in_bytes": 23317672, + "peak_max_in_bytes": 35782656, + "max_in_bytes": 35782656, + "committed_in_bytes": 35782656 + }, + "old": { + "peak_used_in_bytes": 101436232, + "used_in_bytes": 87297304, + "peak_max_in_bytes": 715849728, + "max_in_bytes": 715849728, + "committed_in_bytes": 715849728 + }, + "young": { + "peak_used_in_bytes": 286326784, + "used_in_bytes": 252231712, + "peak_max_in_bytes": 286326784, + "max_in_bytes": 286326784, + "committed_in_bytes": 286326784 + } + } + }, + "gc": { + "collectors": { + "old": { + "collection_time_in_millis": 147, + "collection_count": 2 + }, + "young": { + "collection_time_in_millis": 352, + "collection_count": 8 + } + } + }, + "uptime_in_millis": 118041 + }, + "process": { + "open_file_descriptors": 95, + "peak_open_file_descriptors": 97, + "max_file_descriptors": 10240, + "mem": { + "total_virtual_in_bytes": 7178244096 + }, + "cpu": { + "total_in_millis": 92452, + "percent": 0, + "load_average": { + "1m": 2.6640625 + } + } + }, + "events": { + "in": 0, + "filtered": 0, + "out": 0, + "duration_in_millis": 0, + "queue_push_duration_in_millis": 0 + }, + "pipelines": { + "main": { + "events": { + "duration_in_millis": 0, + "in": 0, + "out": 0, + "filtered": 0, + "queue_push_duration_in_millis": 0 + }, + "plugins": { + "inputs": [ + { + "id": "fc1bd2827edc6b55a3caf4af83c5ac8c5c32b84cb54453bdac8c0d1dcb37ee46", + "events": { + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "name": "stdin" + } + ], + "filters": [], + "outputs": [ + { + "id": "47e3be4432eda99c8d50ea91fdec98aa9c3d5689589ddd3cfdbd62b43746d841", + "events": { + "duration_in_millis": 0, + "in": 0, + "out": 0 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "successes": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "failures": 0 + }, + "queue": { + "type": "memory" + } + } + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "os": {} +} diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.700.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.700.json new file mode 100644 index 00000000000..a3710586d1f --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.700.json @@ -0,0 +1,129 @@ +{ + "host": "Shaunaks-MBP-2", + "version": "7.0.0-alpha1", + "http_address": "127.0.0.1:9600", + "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", + "name": "Shaunaks-MBP-2", + "jvm": { + "threads": { + "count": 30, + "peak_count": 32 + }, + "mem": { + "heap_used_percent": 21, + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 220732000, + "non_heap_used_in_bytes": 139772840, + "non_heap_committed_in_bytes": 158154752, + "pools": { + "survivor": { + "peak_used_in_bytes": 35782656, + "used_in_bytes": 35782648, + "peak_max_in_bytes": 35782656, + "max_in_bytes": 35782656, + "committed_in_bytes": 35782656 + }, + "old": { + "peak_used_in_bytes": 100051832, + "used_in_bytes": 100051832, + "peak_max_in_bytes": 715849728, + "max_in_bytes": 715849728, + "committed_in_bytes": 715849728 + }, + "young": { + "peak_used_in_bytes": 286326784, + "used_in_bytes": 84897520, + "peak_max_in_bytes": 286326784, + "max_in_bytes": 286326784, + "committed_in_bytes": 286326784 + } + } + }, + "gc": { + "collectors": { + "old": { + "collection_time_in_millis": 157, + "collection_count": 2 + }, + "young": { + "collection_time_in_millis": 379, + "collection_count": 9 + } + } + }, + "uptime_in_millis": 242592 + }, + "process": { + "open_file_descriptors": 96, + "peak_open_file_descriptors": 97, + "max_file_descriptors": 10240, + "mem": { + "total_virtual_in_bytes": 7187554304 + }, + "cpu": { + "total_in_millis": 94249, + "percent": 0, + "load_average": { + "1m": 2.5634765625 + } + } + }, + "events": { + "in": 5, + "filtered": 5, + "out": 5, + "duration_in_millis": 331, + "queue_push_duration_in_millis": 0 + }, + "pipelines": { + "main": { + "events": { + "duration_in_millis": 331, + "in": 5, + "out": 5, + "filtered": 5, + "queue_push_duration_in_millis": 0 + }, + "plugins": { + "inputs": [ + { + "id": "fc1bd2827edc6b55a3caf4af83c5ac8c5c32b84cb54453bdac8c0d1dcb37ee46", + "events": { + "out": 5, + "queue_push_duration_in_millis": 0 + }, + "name": "stdin" + } + ], + "filters": [], + "outputs": [ + { + "id": "47e3be4432eda99c8d50ea91fdec98aa9c3d5689589ddd3cfdbd62b43746d841", + "events": { + "duration_in_millis": 2259, + "in": 5, + "out": 5 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "successes": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "failures": 0 + }, + "queue": { + "type": "memory" + } + } + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "os": {} +} diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index 6dad0598b83..6826fccf44d 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -18,6 +18,8 @@ package node_stats import ( + "encoding/json" + "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" @@ -33,6 +35,11 @@ var ( } ) -func eventMapping(node map[string]interface{}) (common.MapStr, error) { - return schema.Apply(node) +func eventMapping(content []byte) (common.MapStr, error) { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return nil, err + } + return schema.Apply(data) } diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go new file mode 100644 index 00000000000..e30aedd33e0 --- /dev/null +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package node_stats + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/node_stats.*.json") + assert.NoError(t, err) + + for _, f := range files { + content, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + event, err := eventMapping(content) + + assert.NoError(t, err, f) + assert.True(t, len(event) >= 1, f) + } +} diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index 0a0671d70db..eb971465c43 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -73,11 +73,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch() (common.MapStr, error) { - data, err := m.http.FetchJSON() + content, err := m.http.FetchContent() if err != nil { return nil, err } - event, _ := eventMapping(data) + event, _ := eventMapping(content) return event, nil }