init commit for avro logger plugin

update to goavro with tcp pool

Signed-off-by: Sébastien GLON <glon.sebastien@free.fr>

daemon/logdrivers_linux.go

@@ -5,6 +5,7 @@ import (
 	// therefore they register themselves to the logdriver factory.
 	_ "github.com/docker/docker/daemon/logger/awslogs"
 	_ "github.com/docker/docker/daemon/logger/fluentd"
+	_ "github.com/docker/docker/daemon/logger/flume"
 	_ "github.com/docker/docker/daemon/logger/gcplogs"
 	_ "github.com/docker/docker/daemon/logger/gelf"
 	_ "github.com/docker/docker/daemon/logger/journald"

daemon/logger/flume/avro.go

@@ -0,0 +1,149 @@
+// Package avro provides th log driver for forwarding server logs to
+// flume endpoints.
+package avro
+
+import (
+	"fmt"
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/daemon/logger"
+	"github.com/sebglon/goavro"
+	"github.com/sebglon/goavro/transceiver"
+	"github.com/sebglon/goavro/transceiver/netty"
+	"net"
+	"strconv"
+)
+
+type avro struct {
+	hostname  string
+	extra     map[string]interface{}
+	conn      *net.TCPConn
+	requestor *goavro.Requestor
+	proto     goavro.Protocol
+}
+
+const (
+	name = "flume-avro"
+
+	defaultHost = "localhost"
+	defaultPort = "63001"
+
+	hostKey = "avro-host"
+	portKey = "avro-port"
+)
+
+func init() {
+	if err := logger.RegisterLogDriver(name, New); err != nil {
+		logrus.Fatal(err)
+	}
+	if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// New create a avro logger using the configuration passed in on
+// the context.
+func New(ctx logger.Context) (logger.Logger, error) {
+	host := ctx.Config[hostKey]
+	port := ctx.Config[portKey]
+
+	logrus.Info("Avro logger socket: " + host + ":" + port)
+
+	// collect extra data for Avro message
+	hostname, err := ctx.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("Avro: cannot access hostname to set source field")
+	}
+
+	extra := map[string]interface{}{
+		"host":              hostname,
+		"daemon_name":       ctx.DaemonName,
+		"container_id":      ctx.ContainerID,
+		"container_name":    ctx.ContainerName,
+		"image_id":          ctx.ContainerImageID,
+		"image_name":        ctx.ContainerImageName,
+		"command":           ctx.Command(),
+		"container_created": strconv.FormatInt(ctx.ContainerCreated.Unix(), 10),
+	}
+
+	for k, v := range ctx.ContainerLabels {
+		extra["container_label_"+k] = v
+	}
+	for k, v := range ctx.ExtraAttributes(nil) {
+		extra["container_meta_"+k] = v
+	}
+
+	proto, err := goavro.NewProtocol()
+	if err != nil {
+		return nil, err
+	}
+
+	iport, err := strconv.Atoi(port)
+	if err != nil {
+		return nil, err
+	}
+
+	transceiver, err := netty.NewTransceiver(transceiver.Config{Host: host, Port: iport})
+	if err != nil {
+		return nil, err
+	}
+
+	requestor := goavro.NewRequestor(proto, transceiver)
+
+	logrus.Info("End init avro plugin")
+	return &avro{
+		extra:     extra,
+		hostname:  hostname,
+		requestor: requestor,
+		proto:     proto,
+	}, nil
+}
+
+func (a *avro) Log(msg *logger.Message) error {
+	flumeRecord, errFlume := a.proto.NewRecord("AvroFlumeEvent")
+	if errFlume != nil {
+		return errFlume
+	}
+	headers := make(map[string]interface{})
+	headers["source"] = msg.Source
+	headers["partial"] = strconv.FormatBool(msg.Partial)
+	headers["timestamp"] = msg.Timestamp.String()
+	for k, v := range a.extra {
+		headers[k] = v
+	}
+	flumeRecord.Set("headers", headers)
+	flumeRecord.Set("body", []byte(msg.Line))
+
+	logrus.WithFields(logrus.Fields{"flumeRecord": flumeRecord}).Info("Avro logger request")
+	err := a.requestor.Request("append", flumeRecord)
+	return err
+}
+
+func (a *avro) Close() error {
+	return a.conn.Close()
+}
+
+func (a *avro) Name() string {
+	return name
+}
+
+// ValidateLogOpt looks for avro specific log option avro-host avro-port.
+func ValidateLogOpt(cfg map[string]string) error {
+	for key := range cfg {
+		switch key {
+		case "env":
+		case "labels":
+		case hostKey:
+		case portKey:
+		//  Accepted
+		default:
+			return fmt.Errorf("unknown log opt '%s' for avro log driver", key)
+		}
+	}
+	if len(cfg[hostKey]) == 0 {
+		cfg[hostKey] = defaultHost
+	}
+	if len(cfg[portKey]) == 0 {
+		cfg[portKey] = defaultPort
+	}
+	return nil
+}

docs/admin/logging/flumeavrologs.md

@@ -0,0 +1,104 @@
+<!--[metadata]>
++++
+aliases = ["/engine/reference/logging/flume/"]
+title = "Flume logging driver"
+description = "Describes how to use the flume logging driver."
+keywords = ["Flume, avro, docker, logging, driver"]
+[menu.main]
+parent = "smn_logging"
++++
+<![end-metadata]-->
+
+# Flume logging driver
+
+The `flume-avor` logging driver send container logs to the [Apache Flume](https://flume.apache.org/) collector as structured log data. Then, isers can use any of the [various sink of Flume](https://flume.apache.org/FlumeUserGuide.html#flume-sinks) to write these logs to various destinations;
+
+In addition to the log message itself, the `flume`log driver sends the following metadata in the structured log message:
+
+| Field            | Description                         |
+-------------------|-------------------------------------|
+| `container_id`   | The full 64-character container ID. |
+| `container_name` | The container name at the time it was started. If you use `docker rename` to rename a container, the new name is not reflected in the journal entries.                                         |
+| `source`         | `stdout` or `stderr`                |
+
+The `docker logs` command is not available for this logging driver.
+
+## usage
+
+Some options are supported by specifying `--log-opt` as many times as needed:
+
+- `avro-host`: specify `host` to connect `localhost`
+- `avro-port`: specify `port` to connect `63001`
+
+Configure the default logging driver by passing the
+`--log-driver` option to the Docker daemon:
+
+    docker daemon --log-driver=flume-avro
+
+To set the logging driver for a specific container, pass the
+`--log-driver` option to `docker run`:
+
+    docker run --log-driver=flume-avro ...
+
+Before using this logging driver, launch a Flume daemon. The logging driver connects to this daemon through `localhost:63001` by default. Use the `flume-host=localhost` and `flume-port=63001` option to connect to a different address.
+
+
+    docker run --log-driver=flume<F4>avro --log-opt flume-host=myhost.local --log-opt flume-port=24224
+
+If container cannot connect to the Flume daemon, the container stops immediately.
+
+## Options
+
+Users can use the `--log-opt NAME=VALUE` flag to specify additional Flume logging driver options.
+
+### avro-host
+
+By default, the logging driver connects to `localhost`. Supply the `avr-host` option to connect to a different address.
+
+### avro-port
+
+By default, the logging driver connects to `63001`. Supply the `avr-port` option to connect to a different port.
+
+
+## Flume daemon management with Docker
+
+About `Apache Flume` itself, see [the project webpage](http://flume.apache.org) and [its documents](http://flume.apache.org/FlumeUserGuide.html).
+
+To use this logging driver,s tart the `flume` daemon on a host. We recommend that use [the Flume docker image](). This 
+
+## Testing container loggers
+
+1. Write a configuration file (`flume.conf`) to dump input logs:
+
+```
+agentLocal.sources = s1
+agentLocal.channels = c1
+agentLocal.sinks = k1
+
+# Channels
+#---------
+
+agentLocal.channels.c1.type = memory
+
+# Source Avro
+#------------
+agentLocal.sources.s1.type=avro
+agentLocal.sources.s1.port=63001
+agentLocal.sources.s1.bind=0.0.0.0
+agentLocal.sources.s1.channels=c1
+agentLocal.sources.s1.byteCapacityBufferPercentage=20
+# Configuration de la destination
+#--------------------------------
+agentLocal.sinks.k1.type = logger
+agentLocal.sinks.k1.channel = c1
+```
+
+
+2. Launch Flume container with this configuration file:
+
+       $ docker run -it -p 63001:63001 -v /path/to/conf/test.conf:/var/tmp/flume.conf -e FLUME_CONF_FILE=/var/tmp/flume.conf -e FLUME_AGENT_NAME=agentLocal probablyfine/flume:latest
+
+3. Start one or more containers with the `flume` logging driver:
+
+        $ docker run --log-driver=flume-avro hello-world
+

docs/admin/logging/overview.md

@@ -31,6 +31,7 @@ supported:
 | `splunk`    | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector.                                 |
 | `etwlogs`   | ETW logging driver for Docker on Windows. Writes log messages as ETW events.                                                  |
 | `gcplogs`   | Google Cloud Logging driver for Docker. Writes log messages to Google Cloud Logging.                                          |
+| `flume-avro`   | Flume Avro Logging driver for Docker. Writes log messages to Apache Flume with Avro protocol.                              |
 
 The `docker logs`command is available only for the `json-file` and `journald`
 logging drivers.
@@ -305,3 +306,32 @@ The Google Cloud Logging driver supports the following options:
 
 For detailed information about working with this logging driver, see the
 [Google Cloud Logging driver](gcplogs.md). reference documentation.
+
+
+## flume-avro options
+
+The Flume Avro logging driver supports the following options:
+
+```bash
+--log-opt avro-host=host
+--log-opt avro-port=port
+--log-opt labels=label1,label2
+--log-opt env=env1,env2
+```
+
+The `avro-host` option specifies the remote Flume server address that the
+driver connects to. 
+
+The `avro-port` option specifies the remote Flume server port that the
+driver connects to. 
+```bash
+$ docker run -dit \
+    --log-driver=flume-avro \
+    --log-opt avro-host=192.168.0.42 \
+    --log-opt avro-port=63001 \
+    alpine sh
+```
+
+By default, Docker uses the first 12 characters of the container ID to tag log
+messages. Refer to the [log tag option documentation](log_tags.md) for
+customizing the log tag format.

hack/vendor.sh

@@ -171,5 +171,9 @@ clone git github.com/spf13/cobra v1.4.1 https://github.com/dnephin/cobra.git
 clone git github.com/spf13/pflag cb88ea77998c3f024757528e3305022ab50b43be
 clone git github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
 clone git github.com/flynn-archive/go-shlex 3f9db97f856818214da2e1057f8ad84803971cff
-
+
+# Avro lig for logger
+clone git github.com/sebglon/goavro FEAT-IPC
+clone git github.com/golang/snappy d9eb7a3d35ec988b8585d4a0068e462c27d28380
+clone git gopkg.in/fatih/pool.v2 v2.0.0
 clean

vendor/src/github.com/golang/snappy/.gitignore

@@ -0,0 +1,16 @@
+cmd/snappytool/snappytool
+testdata/bench
+
+# These explicitly listed benchmark data files are for an obsolete version of
+# snappy_test.go.
+testdata/alice29.txt
+testdata/asyoulik.txt
+testdata/fireworks.jpeg
+testdata/geo.protodata
+testdata/html
+testdata/html_x_4
+testdata/kppkn.gtb
+testdata/lcet10.txt
+testdata/paper-100k.pdf
+testdata/plrabn12.txt
+testdata/urls.10K

vendor/src/github.com/golang/snappy/AUTHORS

@@ -0,0 +1,15 @@
+# This is the official list of Snappy-Go authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Google Inc.
+Jan Mercl <0xjnml@gmail.com>
+Rodolfo Carvalho <rhcarvalho@gmail.com>
+Sebastien Binet <seb.binet@gmail.com>

vendor/src/github.com/golang/snappy/CONTRIBUTORS

@@ -0,0 +1,37 @@
+# This is the official list of people who can contribute
+# (and typically have contributed) code to the Snappy-Go repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Jan Mercl <0xjnml@gmail.com>
+Kai Backman <kaib@golang.org>
+Marc-Antoine Ruel <maruel@chromium.org>
+Nigel Tao <nigeltao@golang.org>
+Rob Pike <r@golang.org>
+Rodolfo Carvalho <rhcarvalho@gmail.com>
+Russ Cox <rsc@golang.org>
+Sebastien Binet <seb.binet@gmail.com>