add ExtJSON (--json flag) mode to wrap messages

app/logger/multiwriter.go

@@ -1,29 +1,66 @@
 package logger
 
 import (
+	"encoding/json"
 	"io"
+	"os"
+	"time"
 
 	"github.com/hashicorp/go-multierror"
 )
 
 // MultiWriter implements WriteCloser for multiple destinations.
 // It is simplified version of stdlib MultiWriter. Ignores write error and don't stop the loop.
 type MultiWriter struct {
-	writers []io.WriteCloser
+	writers   []io.WriteCloser
+	hostname  string
+	container string
+	group     string
+	isExt     bool
+}
+
+// jMsg is envelope for ExtJSON mode
+type jMsg struct {
+	Msg       string    `json:"msg"`
+	Container string    `json:"container"`
+	Group     string    `json:"group"`
+	TS        time.Time `json:"ts"`
+	Host      string    `json:"host"`
 }
 
 // NewMultiWriterIgnoreErrors create WriteCloser for multiple destinations
-func NewMultiWriterIgnoreErrors(writers ...io.WriteCloser) io.WriteCloser {
+func NewMultiWriterIgnoreErrors(writers ...io.WriteCloser) *MultiWriter {
 	w := make([]io.WriteCloser, len(writers))
 	copy(w, writers)
-	return &MultiWriter{w}
+
+	return &MultiWriter{writers: w}
+}
+
+func (w *MultiWriter) WithExtJSON(containerName string, group string) *MultiWriter {
+	w.container = containerName
+	w.group = group
+	w.isExt = true
+
+	hname := "unknown"
+	if h, err := os.Hostname(); err == nil {
+		hname = h
+	}
+	w.hostname = hname
+	return w
 }
 
 // Write to all writers and ignore errors unless they all have errors
 func (w *MultiWriter) Write(p []byte) (n int, err error) {
+	pp := p
+	if w.isExt {
+		if pp, err = w.extJSON(p); err != nil {
+			return 0, err
+		}
+	}
+
 	numErrors := 0
 	for _, w := range w.writers {
-		if _, err = w.Write(p); err != nil {
+		if _, err = w.Write(pp); err != nil {
 			numErrors++
 		}
 	}
@@ -44,3 +81,7 @@ func (w *MultiWriter) Close() error {
 	}
 	return errs.ErrorOrNil()
 }
+
+func (w *MultiWriter) extJSON(p []byte) (res []byte, err error) {
+	return json.Marshal(jMsg{Msg: string(p), TS: time.Now(), Host: w.hostname, Group: w.group, Container: w.container})
+}

app/logger/multiwriter_test.go

@@ -0,0 +1,62 @@
+package logger
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type wrMock struct {
+	bytes.Buffer
+}
+
+func (m *wrMock) Close() error { return nil }
+
+func TestMultiWriter_Write(t *testing.T) {
+
+	// with ext JSON
+	w1, w2 := wrMock{}, wrMock{}
+	writer := NewMultiWriterIgnoreErrors(&w1, &w2).WithExtJSON("c1", "g1")
+	n, err := writer.Write([]byte("test 123"))
+	assert.NoError(t, err)
+	assert.Equal(t, 8, n)
+
+	s1, s2 := w1.String(), w1.String()
+	assert.Equal(t, s1, s2, "both dest writers have the same data")
+	assert.True(t, strings.HasPrefix(w1.String(), `{"msg":"test 123"`))
+	t.Log(s1)
+
+	// without ext JSON
+	w1, w2 = wrMock{}, wrMock{}
+	writer = NewMultiWriterIgnoreErrors(&w1, &w2)
+	n, err = writer.Write([]byte("test 123"))
+	assert.NoError(t, err)
+	assert.Equal(t, 8, n)
+	assert.Equal(t, "test 123", w1.String())
+	assert.Equal(t, "test 123", w2.String())
+}
+
+func TestMultiWriter_extJSON(t *testing.T) {
+
+	writer := NewMultiWriterIgnoreErrors().WithExtJSON("c1", "g1")
+	res, err := writer.extJSON([]byte("test msg"))
+	assert.NoError(t, err)
+
+	j := jMsg{}
+	err = json.Unmarshal(res, &j)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "test msg", j.Msg)
+	assert.Equal(t, "c1", j.Container)
+	assert.Equal(t, "g1", j.Group)
+
+	hname, err := os.Hostname()
+	assert.NoError(t, err)
+	assert.Equal(t, hname, j.Host)
+	assert.True(t, time.Since(j.TS).Seconds() < 1)
+}

app/main.go

@@ -11,7 +11,7 @@ import (
 	"github.com/fsouza/go-dockerclient"
 	"github.com/hashicorp/logutils"
 	"github.com/jessevdk/go-flags"
-	lumberjack "gopkg.in/natefinch/lumberjack.v2"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/umputun/docker-logger/app/discovery"
 	"github.com/umputun/docker-logger/app/logger"
@@ -26,6 +26,7 @@ var opts struct {
 	MaxFilesCount int      `long:"max-files" env:"MAX_FILES" default:"5" description:"number of rotated files to retain"`
 	MaxFilesAge   int      `long:"max-age" env:"MAX_AGE" default:"30" description:"maximum number of days to retain"`
 	Excludes      []string `short:"x" long:"exclude" env:"EXCLUDE" env-delim:"," description:"excluded container names"`
+	ExtJSON       bool     `short:"j" long:"json" env:"JSON" description:"wrap message with JSON envelope"`
 	Dbg           bool     `long:"dbg" env:"DEBUG" description:"debug mode"`
 }
 
@@ -56,7 +57,7 @@ func main() {
 
 		if event.Status {
 			// new/started container detected
-			logWriter, errWriter := MakeLogWriters(event.ContainerName, event.Group)
+			logWriter, errWriter := MakeLogWriters(event.ContainerName, event.Group, opts.ExtJSON)
 			ctx, cancel := context.WithCancel(context.Background())
 			ls := logger.LogStreamer{
 				Context:       ctx,
@@ -92,7 +93,7 @@ func main() {
 }
 
 // MakeLogWriters creates io.Writer with rotated out and separate err files. Also adds writer for remote syslog
-func MakeLogWriters(containerName string, group string) (logWriter, errWriter io.WriteCloser) {
+func MakeLogWriters(containerName string, group string, isExt bool) (logWriter, errWriter io.WriteCloser) {
 	log.Printf("[DEBUG] create log writer for %s/%s", group, containerName)
 	if !opts.EnableFiles && !opts.EnableSyslog {
 		log.Printf("[ERROR] either files or syslog has to be enabled")
@@ -145,7 +146,15 @@ func MakeLogWriters(containerName string, group string) (logWriter, errWriter io
 			log.Printf("[WARN] can't connect to syslog, %v", err)
 		}
 	}
-	return logger.NewMultiWriterIgnoreErrors(logWriters...), logger.NewMultiWriterIgnoreErrors(errWriters...)
+
+	lw := logger.NewMultiWriterIgnoreErrors(logWriters...)
+	ew := logger.NewMultiWriterIgnoreErrors(errWriters...)
+	if isExt {
+		lw = lw.WithExtJSON(containerName, group)
+		ew = ew.WithExtJSON(containerName, group)
+	}
+
+	return lw, ew
 }
 
 func setupLog(dbg bool) {