Fix for rendering ForwardedEvents in Winlogbeat.

When reading events from the ForwardedEvents log, Winlogbeat was always attempting to render the event locally event using messages files on the collector. This was wrong because the collector may not have the same message files as the source machine. It would result in a message_error if the collector machine was missing the application that originally logged the event on the source machine.

This PR changes Winlogbeat to not attempt to render the event on the collector machine, but instead use the event as is. This requires that the subscription for the Windows event collector uses the "RenderedText" format which includes the message string and any description strings with the forwarded event.

CHANGELOG.asciidoc

@@ -45,6 +45,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d
 *Filebeat*
 
 *Winlogbeat*
+- Fix issue with rendering forwarded event log records. {pull}1891[1891]
 
 ==== Added
 

winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc

@@ -122,6 +122,19 @@ winlogbeat.event_logs:
     ignore_older: 168h
 --------------------------------------------------------------------------------
 
+===== event_logs.forwarded
+
+A boolean flag to indicate that the log contains only events collected from
+remote hosts using the Windows Event Collector. The value defaults to true for
+the ForwardedEvents log and false for any other log. *{vista_and_newer}*
+
+This settings allows Winlogbeat to optimize reads for forwarded events that are
+already rendered. When the value is true Winlogbeat does not attempt to render
+the event using message files from the host computer. The Windows Event
+Collector subscription should be configured to use the "RenderedText" format
+(this is the default) to ensure that the events are distributed with messages
+and descriptions.
+
 ===== event_logs.event_id
 
 A whitelist and blacklist of event IDs. The value is a comma-separated list. The

winlogbeat/etc/beat.full.yml

@@ -25,8 +25,8 @@
 # dictionaries.
 #
 # The supported keys are name (required), tags, fields, fields_under_root,
-# ignore_older, level, event_id, provider, and include_xml. Please visit the
-# documentation for the complete details of each option.
+# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
+# visit the documentation for the complete details of each option.
 # https://go.es.io/WinlogbeatConfig
 winlogbeat.event_logs:
   - name: Application

winlogbeat/etc/beat.yml

@@ -14,8 +14,8 @@
 # dictionaries.
 #
 # The supported keys are name (required), tags, fields, fields_under_root,
-# ignore_older, level, event_id, provider, and include_xml. Please visit the
-# documentation for the complete details of each option.
+# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
+# visit the documentation for the complete details of each option.
 # https://go.es.io/WinlogbeatConfig
 winlogbeat.event_logs:
   - name: Application

winlogbeat/eventlog/wineventlog.go

@@ -29,11 +29,12 @@ const (
 )
 
 var winEventLogConfigKeys = append(commonConfigKeys, "ignore_older", "include_xml",
-	"event_id", "level", "provider")
+	"event_id", "forwarded", "level", "provider")
 
 type winEventLogConfig struct {
 	ConfigCommon `config:",inline"`
 	IncludeXML   bool                   `config:"include_xml"`
+	Forwarded    *bool                  `config:"forwarded"`
 	SimpleQuery  query                  `config:",inline"`
 	Raw          map[string]interface{} `config:",inline"`
 }
@@ -70,8 +71,9 @@ type winEventLog struct {
 	subscription win.EvtHandle // Handle to the subscription.
 	maxRead      int           // Maximum number returned in one Read.
 
-	renderBuf []byte             // Buffer used for rendering event.
-	cache     *messageFilesCache // Cached mapping of source name to event message file handles.
+	render    func(event win.EvtHandle) (string, error) // Function for rendering the event to XML.
+	renderBuf []byte                                    // Buffer used for rendering event.
+	cache     *messageFilesCache                        // Cached mapping of source name to event message file handles.
 
 	logPrefix     string               // String to prefix on log messages.
 	eventMetadata common.EventMetadata // Field and tags to add to each event.
@@ -131,12 +133,12 @@ func (l *winEventLog) Read() ([]Record, error) {
 
 	var records []Record
 	for _, h := range handles {
-		x, err := win.RenderEvent(h, 0, l.renderBuf, l.cache.get)
+		x, err := l.render(h)
 		if bufErr, ok := err.(sys.InsufficientBufferError); ok {
 			detailf("%s Increasing render buffer size to %d", l.logPrefix,
 				bufErr.RequiredSize)
 			l.renderBuf = make([]byte, bufErr.RequiredSize)
-			x, err = win.RenderEvent(h, 0, l.renderBuf, l.cache.get)
+			x, err = l.render(h)
 		}
 		if err != nil && x == "" {
 			logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err)
@@ -248,7 +250,7 @@ func newWinEventLog(options map[string]interface{}) (EventLog, error) {
 		return win.Close(win.EvtHandle(handle))
 	}
 
-	return &winEventLog{
+	l := &winEventLog{
 		config:        c,
 		query:         query,
 		channelName:   c.Name,
@@ -257,7 +259,24 @@ func newWinEventLog(options map[string]interface{}) (EventLog, error) {
 		cache:         newMessageFilesCache(c.Name, eventMetadataHandle, freeHandle),
 		logPrefix:     fmt.Sprintf("WinEventLog[%s]", c.Name),
 		eventMetadata: c.EventMetadata,
-	}, nil
+	}
+
+	// Forwarded events should be rendered using RenderEventXML. It is more
+	// efficient and does not attempt to use local message files for rendering
+	// the event's message.
+	switch {
+	case c.Forwarded == nil && c.Name == "ForwardedEvents",
+		c.Forwarded != nil && *c.Forwarded == true:
+		l.render = func(event win.EvtHandle) (string, error) {
+			return win.RenderEventXML(event, l.renderBuf)
+		}
+	default:
+		l.render = func(event win.EvtHandle) (string, error) {
+			return win.RenderEvent(event, 0, l.renderBuf, l.cache.get)
+		}
+	}
+
+	return l, nil
 }
 
 func init() {

winlogbeat/sys/wineventlog/wineventlog_windows.go

@@ -180,15 +180,18 @@ func RenderEvent(
 			return "", err
 		}
 
-		// Ignore the error and return the original error with the response.
-		xml, _ = RenderEventNoMessage(eventHandle, renderBuf)
+		xml, err = RenderEventXML(eventHandle, renderBuf)
 	}
 
 	return xml, err
 }
 
-// RenderEventNoMessage render the events as XML but without the RenderingInfo (message).
-func RenderEventNoMessage(eventHandle EvtHandle, renderBuf []byte) (string, error) {
+// RenderEventXML renders the event as XML. If the event is already rendered, as
+// in a forwarded event whose content type is "RenderedText", then the XML will
+// include the RenderingInfo (message). If the event is not rendered then the
+// XML will not include the message, and in this case RenderEvent should be
+// used.
+func RenderEventXML(eventHandle EvtHandle, renderBuf []byte) (string, error) {
 	var bufferUsed, propertyCount uint32
 	err := _EvtRender(0, eventHandle, EvtRenderEventXml, uint32(len(renderBuf)),
 		&renderBuf[0], &bufferUsed, &propertyCount)

winlogbeat/tests/system/test_wineventlog.py

@@ -279,6 +279,7 @@ def test_unknown_eventlog_config(self):
                 {
                     "name": self.providerName,
                     "api": self.api,
+                    "forwarded": False,
                     "invalid": "garbage"}
             ]
         )

winlogbeat/winlogbeat.full.yml

@@ -25,8 +25,8 @@
 # dictionaries.
 #
 # The supported keys are name (required), tags, fields, fields_under_root,
-# ignore_older, level, event_id, provider, and include_xml. Please visit the
-# documentation for the complete details of each option.
+# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
+# visit the documentation for the complete details of each option.
 # https://go.es.io/WinlogbeatConfig
 winlogbeat.event_logs:
   - name: Application

winlogbeat/winlogbeat.yml

@@ -14,8 +14,8 @@
 # dictionaries.
 #
 # The supported keys are name (required), tags, fields, fields_under_root,
-# ignore_older, level, event_id, provider, and include_xml. Please visit the
-# documentation for the complete details of each option.
+# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
+# visit the documentation for the complete details of each option.
 # https://go.es.io/WinlogbeatConfig
 winlogbeat.event_logs:
   - name: Application