Skip to content

Commit

Permalink
Make logstash module code consistent with other Elastic stack modules (…
Browse files Browse the repository at this point in the history 
…elastic#8497)

This PR makes the Metricbeat `logstash` module code consistent with other Elastic stack modules by:

* using `FetchContent()` instead of `FetchJSON()` to fetch data from LS APIs before passing it to the event mapping function.
* adding unit tests.

Further consistency around event reporting and error handling will be achieved in elastic#8308.
  • Loading branch information
@ycombinator
ycombinator committed Oct 1, 2018
1 parent dd1c408 commit 909d789
Show file tree
Hide file tree
Showing 12 changed files with 621 additions and 8 deletions.
41 changes: 41 additions & 0 deletions metricbeat/module/logstash/node/_meta/test/node.641.json
View file
@@ -0,0 +1,41 @@
{
"host": "Shaunaks-MBP-2",
"version": "6.4.1",
"http_address": "127.0.0.1:9600",
"id": "01cf780e-b576-4e59-9ee0-6b49d73e1238",
"name": "Shaunaks-MBP-2",
"pipelines": {
"main": {
"workers": 8,
"batch_size": 125,
"batch_delay": 50,
"config_reload_automatic": false,
"config_reload_interval": 3000000000,
"dead_letter_queue_enabled": false
}
},
"os": {
"name": "Mac OS X",
"arch": "x86_64",
"version": "10.13.6",
"available_processors": 8
},
"jvm": {
"pid": 92230,
"version": "1.8.0_171",
"vm_version": "1.8.0_171",
"vm_vendor": "Oracle Corporation",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"start_time_in_millis": 1538404190630,
"mem": {
"heap_init_in_bytes": 1073741824,
"heap_max_in_bytes": 1037959168,
"non_heap_init_in_bytes": 2555904,
"non_heap_max_in_bytes": 0
},
"gc_collectors": [
"ParNew",
"ConcurrentMarkSweep"
]
}
}
41 changes: 41 additions & 0 deletions metricbeat/module/logstash/node/_meta/test/node.650.json
View file
@@ -0,0 +1,41 @@
{
"host": "Shaunaks-MBP-2",
"version": "6.5.0",
"http_address": "127.0.0.1:9600",
"id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16",
"name": "Shaunaks-MBP-2",
"pipelines": {
"main": {
"workers": 8,
"batch_size": 125,
"batch_delay": 50,
"config_reload_automatic": false,
"config_reload_interval": 3000000000,
"dead_letter_queue_enabled": false
}
},
"os": {
"name": "Mac OS X",
"arch": "x86_64",
"version": "10.13.6",
"available_processors": 8
},
"jvm": {
"pid": 1559,
"version": "1.8.0_171",
"vm_version": "1.8.0_171",
"vm_vendor": "Oracle Corporation",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"start_time_in_millis": 1538406837613,
"mem": {
"heap_init_in_bytes": 1073741824,
"heap_max_in_bytes": 1037959168,
"non_heap_init_in_bytes": 2555904,
"non_heap_max_in_bytes": 0
},
"gc_collectors": [
"ParNew",
"ConcurrentMarkSweep"
]
}
}
41 changes: 41 additions & 0 deletions metricbeat/module/logstash/node/_meta/test/node.700.json
View file
@@ -0,0 +1,41 @@
{
"host": "Shaunaks-MBP-2",
"version": "7.0.0-alpha1",
"http_address": "127.0.0.1:9600",
"id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16",
"name": "Shaunaks-MBP-2",
"pipelines": {
"main": {
"workers": 8,
"batch_size": 125,
"batch_delay": 50,
"config_reload_automatic": false,
"config_reload_interval": 3000000000,
"dead_letter_queue_enabled": false
}
},
"os": {
"name": "Mac OS X",
"arch": "x86_64",
"version": "10.13.6",
"available_processors": 8
},
"jvm": {
"pid": 83353,
"version": "1.8.0_171",
"vm_version": "1.8.0_171",
"vm_vendor": "Oracle Corporation",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"start_time_in_millis": 1538217473740,
"mem": {
"heap_init_in_bytes": 1073741824,
"heap_max_in_bytes": 1037959168,
"non_heap_init_in_bytes": 2555904,
"non_heap_max_in_bytes": 0
},
"gc_collectors": [
"ParNew",
"ConcurrentMarkSweep"
]
}
}
12 changes: 10 additions & 2 deletions metricbeat/module/logstash/node/data.go
View file
Expand Up @@ -18,6 +18,8 @@
package node

import (
"encoding/json"

"github.com/elastic/beats/libbeat/common"
s "github.com/elastic/beats/libbeat/common/schema"
c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
Expand All @@ -34,6 +36,12 @@ var (
}
)

func eventMapping(node map[string]interface{}) (common.MapStr, error) {
return schema.Apply(node)
func eventMapping(content []byte) (common.MapStr, error) {
var data map[string]interface{}
err := json.Unmarshal(content, &data)
if err != nil {
return nil, err
}

return schema.Apply(data)
}
44 changes: 44 additions & 0 deletions metricbeat/module/logstash/node/data_test.go
View file
@@ -0,0 +1,44 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build !integration

package node

import (
"io/ioutil"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
)

func TestEventMapping(t *testing.T) {

files, err := filepath.Glob("./_meta/test/node.*.json")
assert.NoError(t, err)

for _, f := range files {
content, err := ioutil.ReadFile(f)
assert.NoError(t, err)

event, err := eventMapping(content)

assert.NoError(t, err, f)
assert.True(t, len(event) >= 1, f)
}
}
4 changes: 2 additions & 2 deletions metricbeat/module/logstash/node/node.go
View file
Expand Up @@ -65,11 +65,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// It returns the event which is then forward to the output. In case of an error, a
// descriptive error must be returned.
func (m *MetricSet) Fetch() (common.MapStr, error) {
data, err := m.http.FetchJSON()
content, err := m.http.FetchContent()
if err != nil {
return nil, err
}

event, _ := eventMapping(data)
event, _ := eventMapping(content)
return event, nil
}
129 changes: 129 additions & 0 deletions metricbeat/module/logstash/node_stats/_meta/test/node_stats.641.json
View file
@@ -0,0 +1,129 @@
{
"host": "Shaunaks-MBP-2",
"version": "6.4.1",
"http_address": "127.0.0.1:9600",
"id": "01cf780e-b576-4e59-9ee0-6b49d73e1238",
"name": "Shaunaks-MBP-2",
"jvm": {
"threads": {
"count": 30,
"peak_count": 32
},
"mem": {
"heap_used_percent": 32,
"heap_committed_in_bytes": 1037959168,
"heap_max_in_bytes": 1037959168,
"heap_used_in_bytes": 341759888,
"non_heap_used_in_bytes": 121819632,
"non_heap_committed_in_bytes": 140050432,
"pools": {
"survivor": {
"peak_used_in_bytes": 35782656,
"used_in_bytes": 21321184,
"peak_max_in_bytes": 35782656,
"max_in_bytes": 35782656,
"committed_in_bytes": 35782656
},
"old": {
"peak_used_in_bytes": 103570264,
"used_in_bytes": 91787528,
"peak_max_in_bytes": 715849728,
"max_in_bytes": 715849728,
"committed_in_bytes": 715849728
},
"young": {
"peak_used_in_bytes": 286326784,
"used_in_bytes": 228651176,
"peak_max_in_bytes": 286326784,
"max_in_bytes": 286326784,
"committed_in_bytes": 286326784
}
}
},
"gc": {
"collectors": {
"old": {
"collection_time_in_millis": 184,
"collection_count": 2
},
"young": {
"collection_time_in_millis": 413,
"collection_count": 8
}
}
},
"uptime_in_millis": 70624
},
"process": {
"open_file_descriptors": 95,
"peak_open_file_descriptors": 96,
"max_file_descriptors": 10240,
"mem": {
"total_virtual_in_bytes": 7176814592
},
"cpu": {
"total_in_millis": 100279,
"percent": 0,
"load_average": {
"1m": 5.107421875
}
}
},
"events": {
"in": 0,
"filtered": 0,
"out": 0,
"duration_in_millis": 0,
"queue_push_duration_in_millis": 0
},
"pipelines": {
"main": {
"events": {
"duration_in_millis": 0,
"in": 0,
"out": 0,
"filtered": 0,
"queue_push_duration_in_millis": 0
},
"plugins": {
"inputs": [
{
"id": "fc1bd2827edc6b55a3caf4af83c5ac8c5c32b84cb54453bdac8c0d1dcb37ee46",
"events": {
"out": 0,
"queue_push_duration_in_millis": 0
},
"name": "stdin"
}
],
"filters": [],
"outputs": [
{
"id": "47e3be4432eda99c8d50ea91fdec98aa9c3d5689589ddd3cfdbd62b43746d841",
"events": {
"duration_in_millis": 0,
"in": 0,
"out": 0
},
"name": "stdout"
}
]
},
"reloads": {
"last_error": null,
"successes": 0,
"last_success_timestamp": null,
"last_failure_timestamp": null,
"failures": 0
},
"queue": {
"type": "memory"
}
}
},
"reloads": {
"successes": 0,
"failures": 0
},
"os": {}
}

0 comments on commit 909d789

Please sign in to comment.