From 5c305943798ff6b6ae22619ce8bafb5ea86a886a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 16 Jun 2016 17:13:26 -0400 Subject: [PATCH] Fix for rendering ForwardedEvents in Winlogbeat. When reading events from the ForwardedEvents log, Winlogbeat was always attempting to render the event locally event using messages files on the collector. This was wrong because the collector may not have the same message files as the source machine. It would result in a message_error if the collector machine was missing the application that originally logged the event on the source machine. This PR changes Winlogbeat to not attempt to render the event on the collector machine, but instead use the event as is. This requires that the subscription for the Windows event collector uses the "RenderedText" format which includes the message string and any description strings with the forwarded event. --- CHANGELOG.asciidoc | 1 + .../configuration/winlogbeat-options.asciidoc | 13 ++++++++ winlogbeat/etc/beat.full.yml | 4 +-- winlogbeat/etc/beat.yml | 4 +-- winlogbeat/eventlog/wineventlog.go | 33 +++++++++++++++---- .../sys/wineventlog/wineventlog_windows.go | 11 ++++--- winlogbeat/tests/system/test_wineventlog.py | 1 + winlogbeat/winlogbeat.full.yml | 4 +-- winlogbeat/winlogbeat.yml | 4 +-- 9 files changed, 56 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0d653e2e142..b1669de6b99 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -45,6 +45,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d *Filebeat* *Winlogbeat* +- Fix issue with rendering forwarded event log records. {pull}1891[1891] ==== Added diff --git a/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc b/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc index 3f9c5749b3e..b2192f5f3f7 100644 --- a/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc +++ b/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc @@ -122,6 +122,19 @@ winlogbeat.event_logs: ignore_older: 168h -------------------------------------------------------------------------------- +===== event_logs.forwarded + +A boolean flag to indicate that the log contains only events collected from +remote hosts using the Windows Event Collector. The value defaults to true for +the ForwardedEvents log and false for any other log. *{vista_and_newer}* + +This settings allows Winlogbeat to optimize reads for forwarded events that are +already rendered. When the value is true Winlogbeat does not attempt to render +the event using message files from the host computer. The Windows Event +Collector subscription should be configured to use the "RenderedText" format +(this is the default) to ensure that the events are distributed with messages +and descriptions. + ===== event_logs.event_id A whitelist and blacklist of event IDs. The value is a comma-separated list. The diff --git a/winlogbeat/etc/beat.full.yml b/winlogbeat/etc/beat.full.yml index f08d68fb985..50e3f104600 100644 --- a/winlogbeat/etc/beat.full.yml +++ b/winlogbeat/etc/beat.full.yml @@ -25,8 +25,8 @@ # dictionaries. # # The supported keys are name (required), tags, fields, fields_under_root, -# ignore_older, level, event_id, provider, and include_xml. Please visit the -# documentation for the complete details of each option. +# forwarded, ignore_older, level, event_id, provider, and include_xml. Please +# visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig winlogbeat.event_logs: - name: Application diff --git a/winlogbeat/etc/beat.yml b/winlogbeat/etc/beat.yml index 0c08cefb44d..d40f326e7d6 100644 --- a/winlogbeat/etc/beat.yml +++ b/winlogbeat/etc/beat.yml @@ -14,8 +14,8 @@ # dictionaries. # # The supported keys are name (required), tags, fields, fields_under_root, -# ignore_older, level, event_id, provider, and include_xml. Please visit the -# documentation for the complete details of each option. +# forwarded, ignore_older, level, event_id, provider, and include_xml. Please +# visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig winlogbeat.event_logs: - name: Application diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 183129ce38b..154cbc83d0d 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -29,11 +29,12 @@ const ( ) var winEventLogConfigKeys = append(commonConfigKeys, "ignore_older", "include_xml", - "event_id", "level", "provider") + "event_id", "forwarded", "level", "provider") type winEventLogConfig struct { ConfigCommon `config:",inline"` IncludeXML bool `config:"include_xml"` + Forwarded *bool `config:"forwarded"` SimpleQuery query `config:",inline"` Raw map[string]interface{} `config:",inline"` } @@ -70,8 +71,9 @@ type winEventLog struct { subscription win.EvtHandle // Handle to the subscription. maxRead int // Maximum number returned in one Read. - renderBuf []byte // Buffer used for rendering event. - cache *messageFilesCache // Cached mapping of source name to event message file handles. + render func(event win.EvtHandle) (string, error) // Function for rendering the event to XML. + renderBuf []byte // Buffer used for rendering event. + cache *messageFilesCache // Cached mapping of source name to event message file handles. logPrefix string // String to prefix on log messages. eventMetadata common.EventMetadata // Field and tags to add to each event. @@ -131,12 +133,12 @@ func (l *winEventLog) Read() ([]Record, error) { var records []Record for _, h := range handles { - x, err := win.RenderEvent(h, 0, l.renderBuf, l.cache.get) + x, err := l.render(h) if bufErr, ok := err.(sys.InsufficientBufferError); ok { detailf("%s Increasing render buffer size to %d", l.logPrefix, bufErr.RequiredSize) l.renderBuf = make([]byte, bufErr.RequiredSize) - x, err = win.RenderEvent(h, 0, l.renderBuf, l.cache.get) + x, err = l.render(h) } if err != nil && x == "" { logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err) @@ -248,7 +250,7 @@ func newWinEventLog(options map[string]interface{}) (EventLog, error) { return win.Close(win.EvtHandle(handle)) } - return &winEventLog{ + l := &winEventLog{ config: c, query: query, channelName: c.Name, @@ -257,7 +259,24 @@ func newWinEventLog(options map[string]interface{}) (EventLog, error) { cache: newMessageFilesCache(c.Name, eventMetadataHandle, freeHandle), logPrefix: fmt.Sprintf("WinEventLog[%s]", c.Name), eventMetadata: c.EventMetadata, - }, nil + } + + // Forwarded events should be rendered using RenderEventXML. It is more + // efficient and does not attempt to use local message files for rendering + // the event's message. + switch { + case c.Forwarded == nil && c.Name == "ForwardedEvents", + c.Forwarded != nil && *c.Forwarded == true: + l.render = func(event win.EvtHandle) (string, error) { + return win.RenderEventXML(event, l.renderBuf) + } + default: + l.render = func(event win.EvtHandle) (string, error) { + return win.RenderEvent(event, 0, l.renderBuf, l.cache.get) + } + } + + return l, nil } func init() { diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index bc5eab1c454..f4f38452428 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -180,15 +180,18 @@ func RenderEvent( return "", err } - // Ignore the error and return the original error with the response. - xml, _ = RenderEventNoMessage(eventHandle, renderBuf) + xml, err = RenderEventXML(eventHandle, renderBuf) } return xml, err } -// RenderEventNoMessage render the events as XML but without the RenderingInfo (message). -func RenderEventNoMessage(eventHandle EvtHandle, renderBuf []byte) (string, error) { +// RenderEventXML renders the event as XML. If the event is already rendered, as +// in a forwarded event whose content type is "RenderedText", then the XML will +// include the RenderingInfo (message). If the event is not rendered then the +// XML will not include the message, and in this case RenderEvent should be +// used. +func RenderEventXML(eventHandle EvtHandle, renderBuf []byte) (string, error) { var bufferUsed, propertyCount uint32 err := _EvtRender(0, eventHandle, EvtRenderEventXml, uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, &propertyCount) diff --git a/winlogbeat/tests/system/test_wineventlog.py b/winlogbeat/tests/system/test_wineventlog.py index 37562dfa9f5..28e2f99f4de 100644 --- a/winlogbeat/tests/system/test_wineventlog.py +++ b/winlogbeat/tests/system/test_wineventlog.py @@ -279,6 +279,7 @@ def test_unknown_eventlog_config(self): { "name": self.providerName, "api": self.api, + "forwarded": False, "invalid": "garbage"} ] ) diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 202567f6797..dbe63553697 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -25,8 +25,8 @@ # dictionaries. # # The supported keys are name (required), tags, fields, fields_under_root, -# ignore_older, level, event_id, provider, and include_xml. Please visit the -# documentation for the complete details of each option. +# forwarded, ignore_older, level, event_id, provider, and include_xml. Please +# visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig winlogbeat.event_logs: - name: Application diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 64863b16f5b..109332e9242 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -14,8 +14,8 @@ # dictionaries. # # The supported keys are name (required), tags, fields, fields_under_root, -# ignore_older, level, event_id, provider, and include_xml. Please visit the -# documentation for the complete details of each option. +# forwarded, ignore_older, level, event_id, provider, and include_xml. Please +# visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig winlogbeat.event_logs: - name: Application