diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index fdf43bcdcdb..837cb9caf88 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -18,6 +18,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Mark `system.syslog.message` and `system.auth.message` as `text` instead of `keyword`. {pull}6589[6589] - Allow override of dynamic template `match_mapping_type` for fields with object_type. {pull}6691[6691] - Set default kafka version to 1.0.0 in kafka output. Older versions are still supported by configuring the `version` setting. {pull}7025[7025] +- Add `host.name` field to all events, to avoid mapping conflicts. This could be breaking Logstash configs if you rely on the `host` field being a string. {pull}7051[7051] *Auditbeat* diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index 35e7ee8d362..4db34c03bc8 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -36,13 +36,14 @@ exec { go get -u github.com/jstemmer/go-junit-report } echo "Building $env:beat" exec { go build } "Build FAILURE" +# always build the libbeat fields +cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml +cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml +cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml + if ($env:beat -eq "metricbeat") { cp .\_meta\fields.common.yml .\_meta\fields.generated.yml python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml -} elseif ($env:beat -eq "libbeat") { - cp .\_meta\fields.common.yml .\_meta\fields.generated.yml - cat processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath .\_meta\fields.generated.yml - cp .\_meta\fields.generated.yml .\fields.yml } echo "Unit testing $env:beat" diff --git a/libbeat/publisher/pipeline/module.go b/libbeat/publisher/pipeline/module.go index 594ce572a17..6bcefa0a28e 100644 --- a/libbeat/publisher/pipeline/module.go +++ b/libbeat/publisher/pipeline/module.go @@ -50,10 +50,15 @@ func Load( Processors: processors, Annotations: Annotations{ Event: config.EventMetadata, - Beat: common.MapStr{ - "name": name, - "hostname": beatInfo.Hostname, - "version": beatInfo.Version, + Builtin: common.MapStr{ + "beat": common.MapStr{ + "name": name, + "hostname": beatInfo.Hostname, + "version": beatInfo.Version, + }, + "host": common.MapStr{ + "name": name, + }, }, }, } diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index fb468d2a074..8ffa191180a 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -62,9 +62,9 @@ type pipelineProcessors struct { // The pipeline its processor settings for // constructing the clients complete processor // pipeline on connect. - beatsMeta common.MapStr - fields common.MapStr - tags []string + builtinMeta common.MapStr + fields common.MapStr + tags []string processors beat.Processor @@ -91,8 +91,8 @@ type Settings struct { // processors, so all processors configured with the pipeline or client will see // the same/complete event. type Annotations struct { - Beat common.MapStr - Event common.EventMetadata + Event common.EventMetadata + Builtin common.MapStr } // WaitCloseMode enumerates the possible behaviors of WaitClose in a pipeline. @@ -401,8 +401,8 @@ func makePipelineProcessors( p.processors = tmp } - if meta := annotations.Beat; meta != nil { - p.beatsMeta = common.MapStr{"beat": meta} + if meta := annotations.Builtin; meta != nil { + p.builtinMeta = meta } if em := annotations.Event; len(em.Fields) > 0 { diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index e8e68fbbd14..5f058cdde81 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -99,8 +99,8 @@ func newProcessorPipeline( // setup 5: client processor list processors.add(localProcessors) - // setup 6: add beats metadata - if meta := global.beatsMeta; len(meta) > 0 { + // setup 6: add beats and host metadata + if meta := global.builtinMeta; len(meta) > 0 { processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy)) } diff --git a/metricbeat/tests/system/metricbeat.py b/metricbeat/tests/system/metricbeat.py index f46d8fabf99..74fe07c7ab7 100644 --- a/metricbeat/tests/system/metricbeat.py +++ b/metricbeat/tests/system/metricbeat.py @@ -7,7 +7,7 @@ from beat.beat import TestCase COMMON_FIELDS = ["@timestamp", "beat", "metricset.name", "metricset.host", - "metricset.module", "metricset.rtt"] + "metricset.module", "metricset.rtt", "host.name"] INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) diff --git a/metricbeat/tests/system/test_processors.py b/metricbeat/tests/system/test_processors.py index 840a7fefdba..e9192d092d9 100644 --- a/metricbeat/tests/system/test_processors.py +++ b/metricbeat/tests/system/test_processors.py @@ -35,7 +35,7 @@ def test_drop_fields(self): print(evt.keys()) self.assertItemsEqual(self.de_dot([ 'beat', '@timestamp', 'system', 'metricset.module', - 'metricset.rtt', 'metricset.name' + 'metricset.rtt', 'metricset.name', 'host' ]), evt.keys()) cpu = evt["system"]["cpu"] print(cpu.keys())