Add the ability to set custom fields in shipper config

Add the ability to set tags in Filebeat prospector config
Add the ability to set tags and fields in Winlogbeat event log config
Update docs to remove wording about all scalars being changed to strings.

Closes #726

CHANGELOG.asciidoc

@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 
 *Filebeat*
 - Default config for ignore_older is now infinite instead of 24h, means ignore_older is disabled by default. Use close_older to only close file handlers.
+- Scalar values in used in the `fields` configuration setting are no longer automatically converted to strings. {pull}1092[1092]
 
 *Winlogbeat*
 
@@ -72,6 +73,7 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 - Improve logstash and elasticsearch backoff behavior. {pull}927[927]
 - Add experimental Kafka output. {pull}942[942]
 - Add config file option to configure GOMAXPROCS. {pull}969[969]
+- Added a `fields` and `fields_under_root` as options available under the `shipper` configuration {pull}1092[1092]
 
 *Packetbeat*
 - Change the DNS library used throughout the dns package to github.com/miekg/dns. {pull}803[803]
@@ -84,10 +86,12 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 - Add multiline support for combining multiple related lines into one event. {issue}461[461]
 - Add close_older configuration option to complete ignore_older https://github.com/elastic/filebeat/issues/181[181]
 - Add experimental option to enable filebeat publisher pipeline to operate asynchonrously {pull}782[782]
+- Added the ability to set a list of tags for each prospector {pull}1092[1092]
 
 *Winlogbeat*
 - Add caching of event metadata handles and the system render context for the wineventlog API {pull}888[888]
 - Improve config validation by checking for unknown top-level YAML keys. {pull}1100[1100]
+- Added the ability to set tags, fields, and fields_under_root as options for each event log {pull}1092[1092]
 
 ==== Deprecated
 

filebeat/config/config.go

@@ -57,11 +57,11 @@ type ProspectorConfig struct {
 }
 
 type HarvesterConfig struct {
+	common.EventMetadata `config:",inline"` // Fields and tags to add to events.
+
 	BufferSize         int    `config:"harvester_buffer_size"`
 	DocumentType       string `config:"document_type"`
 	Encoding           string `config:"encoding"`
-	Fields             common.MapStr
-	FieldsUnderRoot    bool   `config:"fields_under_root"`
 	InputType          string `config:"input_type"`
 	TailFiles          bool   `config:"tail_files"`
 	Backoff            string `config:"backoff"`

filebeat/docs/reference/configuration/filebeat-options.asciidoc

@@ -92,27 +92,51 @@ The following example configures Filebeat to ignore all the files that have a `g
   exclude_files: [".gz$"]
 -------------------------------------------------------------------------------------
 
+===== tags
+
+A list of tags that the Beat includes in the `tags` field of each published
+event. Tags make it easy to select specific events in Kibana or apply
+conditional filtering in Logstash. These tags will be appended to the list of
+tags specified in the `shipper` configuration.
+
+Example:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+filebeat:
+  prospectors:
+    - paths: ["/var/log/app/*.json"]
+      tags: ["json"]
+--------------------------------------------------------------------------------
+
 [[configuration-fields]]
 ===== fields
 
-Optional fields that you can specify to add additional information to the output. For
-example, you might add fields that you can use for filtering log data. Fields can be
-scalar values, arrays, dictionaries, or any nested combination of these. All scalar values will be interpreted as strings. By default,
-the fields that you specify here will be grouped under a `fields` sub-dictionary in the output document. To store the custom fields as top-level fields, set the `fields_under_root` option to true.
+Optional fields that you can specify to add additional information to the
+output. For example, you might add fields that you can use for filtering log
+data. Fields can be scalar values, arrays, dictionaries, or any nested
+combination of these. By default, the fields that you specify here will be
+grouped under a `fields` sub-dictionary in the output document. To store the
+custom fields as top-level fields, set the `fields_under_root` option to true.
+If a duplicate field is declared in the `shipper` configuration, then its value
+will be overwritten by the value declared here.
 
 [source,yaml]
--------------------------------------------------------------------------------------
-fields:
-    level: debug
-    review: 1
+--------------------------------------------------------------------------------
+filebeat:
+  prospectors:
+    - paths: ["/var/log/app/*.log"]
+      fields:
+        app_id: query_engine_12
+--------------------------------------------------------------------------------
 
--------------------------------------------------------------------------------------
 [[fields-under-root]]
 ===== fields_under_root
 
-If this option is set to true, the custom <<configuration-fields>> are stored as top-level fields
-in the output document instead of being grouped under a `fields` sub-dictionary.
-If the custom field names conflict with other field names added by Filebeat, the custom fields overwrite the other fields.
+If this option is set to true, the custom <<configuration-fields>> are stored as
+top-level fields in the output document instead of being grouped under a
+`fields` sub-dictionary. If the custom field names conflict with other field
+names added by Filebeat, then the custom fields overwrite the other fields.
 
 [[ignore-older]]
 ===== ignore_older

filebeat/filebeat.yml

@@ -360,6 +360,17 @@ shipper:
   # logical properties.
   #tags: ["service-X", "web-tier"]
 
+  # Optional fields that you can specify to add additional information to the
+  # output. Fields can be scalar values, arrays, dictionaries, or any nested
+  # combination of these. 
+  #fields:
+  #  env: staging
+
+  # If this option is set to true, the custom fields are stored as top-level
+  # fields in the output document instead of being grouped under a fields
+  # sub-dictionary. Default is false.
+  #fields_under_root: false
+
   # Uncomment the following if you want to ignore transactions created
   # by the server on which the shipper is installed. This option is useful
   # to remove duplicates if shippers are installed on multiple servers.

filebeat/harvester/log.go

@@ -127,20 +127,18 @@ func (h *Harvester) Harvest() {
 		}
 
 		if h.shouldExportLine(text) {
-			// Sends text to spooler
 			event := &input.FileEvent{
-				ReadTime:     ts,
-				Source:       &h.Path,
-				InputType:    h.Config.InputType,
-				DocumentType: h.Config.DocumentType,
-				Offset:       h.Offset,
-				Bytes:        bytesRead,
-				Text:         &text,
-				Fields:       h.Config.Fields,
-				Fileinfo:     &info,
+				EventMetadata: h.Config.EventMetadata,
+				ReadTime:      ts,
+				Source:        &h.Path,
+				InputType:     h.Config.InputType,
+				DocumentType:  h.Config.DocumentType,
+				Offset:        h.Offset,
+				Bytes:         bytesRead,
+				Text:          &text,
+				Fileinfo:      &info,
 			}
 
-			event.SetFieldsUnderRoot(h.Config.FieldsUnderRoot)
 			h.SpoolerChan <- event // ship the new event downstream
 		}
 

filebeat/input/file.go

@@ -17,17 +17,15 @@ type File struct {
 
 // FileEvent is sent to the output and must contain all relevant information
 type FileEvent struct {
+	common.EventMetadata
 	ReadTime     time.Time
 	Source       *string
 	InputType    string
 	DocumentType string
 	Offset       int64
 	Bytes        int
 	Text         *string
-	Fields       common.MapStr
 	Fileinfo     *os.FileInfo
-
-	fieldsUnderRoot bool
 }
 
 type FileState struct {
@@ -57,37 +55,16 @@ func (f *FileEvent) GetState() *FileState {
 	return state
 }
 
-// SetFieldsUnderRoot sets whether the fields should be added
-// top level to the output documentation (fieldsUnderRoot = true) or
-// under a fields dictionary.
-func (f *FileEvent) SetFieldsUnderRoot(fieldsUnderRoot bool) {
-	f.fieldsUnderRoot = fieldsUnderRoot
-}
-
 func (f *FileEvent) ToMapStr() common.MapStr {
 	event := common.MapStr{
-		"@timestamp": common.Time(f.ReadTime),
-		"source":     f.Source,
-		"offset":     f.Offset, // Offset here is the offset before the starting char.
-		"message":    f.Text,
-		"type":       f.DocumentType,
-		"input_type": f.InputType,
-		"count":      1,
-	}
-
-	if f.Fields != nil {
-		if f.fieldsUnderRoot {
-			for key, value := range f.Fields {
-				// in case of conflicts, overwrite
-				_, found := event[key]
-				if found {
-					logp.Warn("Overwriting %s key", key)
-				}
-				event[key] = value
-			}
-		} else {
-			event["fields"] = f.Fields
-		}
+		common.EventMetadataKey: f.EventMetadata,
+		"@timestamp":            common.Time(f.ReadTime),
+		"source":                f.Source,
+		"offset":                f.Offset, // Offset here is the offset before the starting char.
+		"message":               f.Text,
+		"type":                  f.DocumentType,
+		"input_type":            f.InputType,
+		"count":                 1,
 	}
 
 	return event

filebeat/input/file_test.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/elastic/beats/libbeat/common"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -112,23 +111,3 @@ func TestFileEventToMapStr(t *testing.T) {
 	_, found := mapStr["fields"]
 	assert.False(t, found)
 }
-
-func TestFieldsUnderRoot(t *testing.T) {
-	event := FileEvent{
-		Fields: common.MapStr{
-			"hello": "world",
-		},
-	}
-	event.SetFieldsUnderRoot(true)
-	mapStr := event.ToMapStr()
-	_, found := mapStr["fields"]
-	assert.False(t, found)
-	assert.Equal(t, "world", mapStr["hello"])
-
-	event.SetFieldsUnderRoot(false)
-	mapStr = event.ToMapStr()
-	_, found = mapStr["hello"]
-	assert.False(t, found)
-	_, found = mapStr["fields"]
-	assert.True(t, found)
-}

filebeat/tests/system/test_fields.py

@@ -15,7 +15,7 @@ def test_custom_fields(self):
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/test.log",
-            fields={"hello": "world"}
+            fields={"hello": "world", "number": 2}
         )
 
         with open(self.working_dir + "/test.log", "w") as f:
@@ -28,6 +28,7 @@ def test_custom_fields(self):
         output = self.read_output()
         doc = output[0]
         assert doc["fields.hello"] == "world"
+        assert doc["fields.number"] == 2
 
     def test_custom_fields_under_root(self):
         """