Add support for json logging to file

CHANGELOG.asciidoc

@@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 - New cli subcommands interface. {pull}4420[4420]
 - Allow source path matching in `add_docker_metadata` processor. {pull}4495[4495]
 - Add support for analyzers and multifields in fields.yml. {pull}4574[4574]
+- Add support for JSON logging. {pull}4523[4523]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -733,3 +733,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

filebeat/filebeat.reference.yml

@@ -1115,3 +1115,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

heartbeat/heartbeat.reference.yml

@@ -888,3 +888,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

libbeat/_meta/config.reference.yml

@@ -674,3 +674,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

libbeat/logp/log.go

@@ -1,10 +1,12 @@
 package logp
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"runtime/debug"
+	"strings"
 	"time"
 )
 
@@ -30,6 +32,7 @@ type logger struct {
 	level             Priority
 	selectors         map[string]struct{}
 	debugAllSelectors bool
+	JSON              bool
 
 	logger  *log.Logger
 	syslog  [LOG_DEBUG + 1]*log.Logger
@@ -73,21 +76,50 @@ func parseSelectors(selectors []string) (map[string]struct{}, bool) {
 
 func debugMessage(calldepth int, selector, format string, v ...interface{}) {
 	if _log.level >= LOG_DEBUG && IsDebug(selector) {
-		send(calldepth+1, LOG_DEBUG, "DBG  ", format, v...)
+		send(calldepth+1, LOG_DEBUG, "DBG", format, v...)
 	}
 }
 
 func send(calldepth int, level Priority, prefix string, format string, v ...interface{}) {
+
+	message := fmt.Sprintf(format, v...)
+	timestamp := time.Now().Format(time.RFC3339)
+	var bytes []byte
+	if _log.JSON {
+		log := map[string]interface{}{
+			"timestamp": timestamp,
+			"level":     prefix,
+			"message":   message,
+		}
+		bytes, _ = json.Marshal(log)
+	} else {
+		// Creates the log message and formats it
+		bytes = []byte(fmt.Sprintf("%s %s %s", timestamp, prefix, message))
+	}
+
 	if _log.toSyslog {
-		_log.syslog[level].Output(calldepth, fmt.Sprintf(format, v...))
+		if _log.JSON {
+			_log.syslog[level].Output(calldepth, string(bytes))
+		} else {
+			_log.syslog[level].Output(calldepth, string(message))
+		}
 	}
 	if _log.toStderr {
-		_log.logger.Output(calldepth, fmt.Sprintf(prefix+format, v...))
+		if _log.JSON {
+			_log.logger.Output(calldepth, string(bytes))
+		} else {
+			_log.logger.Output(calldepth, fmt.Sprintf("%s %s", prefix, message))
+		}
 	}
 	if _log.toFile {
-		// Creates a timestamp for the file log message and formats it
-		prefix = time.Now().Format(time.RFC3339) + " " + prefix
-		_log.rotator.WriteLine([]byte(fmt.Sprintf(prefix+format, v...)))
+		if _log.JSON {
+			_log.rotator.WriteLine(bytes)
+		} else {
+			// Makes sure all prefixes have the same length
+			prefix = prefix + strings.Repeat(" ", 4-len(prefix))
+			bytes = []byte(fmt.Sprintf("%s %s %s", timestamp, prefix, message))
+			_log.rotator.WriteLine(bytes)
+		}
 	}
 }
 
@@ -117,19 +149,19 @@ func msg(level Priority, prefix string, format string, v ...interface{}) {
 }
 
 func Info(format string, v ...interface{}) {
-	msg(LOG_INFO, "INFO ", format, v...)
+	msg(LOG_INFO, "INFO", format, v...)
 }
 
 func Warn(format string, v ...interface{}) {
-	msg(LOG_WARNING, "WARN ", format, v...)
+	msg(LOG_WARNING, "WARN", format, v...)
 }
 
 func Err(format string, v ...interface{}) {
-	msg(LOG_ERR, "ERR ", format, v...)
+	msg(LOG_ERR, "ERR", format, v...)
 }
 
 func Critical(format string, v ...interface{}) {
-	msg(LOG_CRIT, "CRIT ", format, v...)
+	msg(LOG_CRIT, "CRIT", format, v...)
 }
 
 // Deprecate logs a deprecation message.
@@ -152,7 +184,7 @@ func Beta(format string, v ...interface{}) {
 // WTF prints the message at CRIT level and panics immediately with the same
 // message
 func WTF(format string, v ...interface{}) {
-	msg(LOG_CRIT, "CRIT ", format, v)
+	msg(LOG_CRIT, "CRIT", format, v)
 	panic(fmt.Sprintf(format, v...))
 }
 

libbeat/logp/logp.go

@@ -27,6 +27,7 @@ type Logging struct {
 	Files     *FileRotator
 	ToSyslog  *bool `config:"to_syslog"`
 	ToFiles   *bool `config:"to_files"`
+	JSON      bool  `config:"json"`
 	Level     string
 	Metrics   LoggingMetricsConfig `config:"metrics"`
 }
@@ -47,7 +48,6 @@ func init() {
 	verbose = flag.Bool("v", false, "Log at INFO level")
 	toStderr = flag.Bool("e", false, "Log to stderr and disable syslog/file output")
 	debugSelectorsStr = flag.String("d", "", "Enable certain debug selectors")
-
 }
 
 func HandleFlags(name string) error {
@@ -80,7 +80,9 @@ func HandleFlags(name string) error {
 // line flag with a later SetStderr call.
 func Init(name string, config *Logging) error {
 	// reset settings from HandleFlags
-	_log = logger{}
+	_log = logger{
+		JSON: config.JSON,
+	}
 
 	logLevel, err := getLogLevel(config)
 	if err != nil {

metricbeat/metricbeat.reference.yml

@@ -1112,3 +1112,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

packetbeat/packetbeat.reference.yml

@@ -1129,3 +1129,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

winlogbeat/winlogbeat.reference.yml

@@ -703,3 +703,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false