-
Notifications
You must be signed in to change notification settings - Fork 18.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
update to goavro with tcp pool Signed-off-by: Sébastien GLON <glon.sebastien@free.fr>
- Loading branch information
dev
committed
Sep 8, 2016
1 parent
4219c00
commit b69d612
Showing
47 changed files
with
7,265 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,149 @@ | ||
// Package avro provides th log driver for forwarding server logs to | ||
// flume endpoints. | ||
package avro | ||
|
||
import ( | ||
"fmt" | ||
"github.com/Sirupsen/logrus" | ||
"github.com/docker/docker/daemon/logger" | ||
"github.com/sebglon/goavro" | ||
"github.com/sebglon/goavro/transceiver" | ||
"github.com/sebglon/goavro/transceiver/netty" | ||
"net" | ||
"strconv" | ||
) | ||
|
||
type avro struct { | ||
hostname string | ||
extra map[string]interface{} | ||
conn *net.TCPConn | ||
requestor *goavro.Requestor | ||
proto goavro.Protocol | ||
} | ||
|
||
const ( | ||
name = "flume-avro" | ||
|
||
defaultHost = "localhost" | ||
defaultPort = "63001" | ||
|
||
hostKey = "avro-host" | ||
portKey = "avro-port" | ||
) | ||
|
||
func init() { | ||
if err := logger.RegisterLogDriver(name, New); err != nil { | ||
logrus.Fatal(err) | ||
} | ||
if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil { | ||
logrus.Fatal(err) | ||
} | ||
} | ||
|
||
// New create a avro logger using the configuration passed in on | ||
// the context. | ||
func New(ctx logger.Context) (logger.Logger, error) { | ||
host := ctx.Config[hostKey] | ||
port := ctx.Config[portKey] | ||
|
||
logrus.Info("Avro logger socket: " + host + ":" + port) | ||
|
||
// collect extra data for Avro message | ||
hostname, err := ctx.Hostname() | ||
if err != nil { | ||
return nil, fmt.Errorf("Avro: cannot access hostname to set source field") | ||
} | ||
|
||
extra := map[string]interface{}{ | ||
"host": hostname, | ||
"daemon_name": ctx.DaemonName, | ||
"container_id": ctx.ContainerID, | ||
"container_name": ctx.ContainerName, | ||
"image_id": ctx.ContainerImageID, | ||
"image_name": ctx.ContainerImageName, | ||
"command": ctx.Command(), | ||
"container_created": strconv.FormatInt(ctx.ContainerCreated.Unix(), 10), | ||
} | ||
|
||
for k, v := range ctx.ContainerLabels { | ||
extra["container_label_"+k] = v | ||
} | ||
for k, v := range ctx.ExtraAttributes(nil) { | ||
extra["container_meta_"+k] = v | ||
} | ||
|
||
proto, err := goavro.NewProtocol() | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
iport, err := strconv.Atoi(port) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
transceiver, err := netty.NewTransceiver(transceiver.Config{Host: host, Port: iport}) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
requestor := goavro.NewRequestor(proto, transceiver) | ||
|
||
logrus.Info("End init avro plugin") | ||
return &avro{ | ||
extra: extra, | ||
hostname: hostname, | ||
requestor: requestor, | ||
proto: proto, | ||
}, nil | ||
} | ||
|
||
func (a *avro) Log(msg *logger.Message) error { | ||
flumeRecord, errFlume := a.proto.NewRecord("AvroFlumeEvent") | ||
if errFlume != nil { | ||
return errFlume | ||
} | ||
headers := make(map[string]interface{}) | ||
headers["source"] = msg.Source | ||
headers["partial"] = strconv.FormatBool(msg.Partial) | ||
headers["timestamp"] = msg.Timestamp.String() | ||
for k, v := range a.extra { | ||
headers[k] = v | ||
} | ||
flumeRecord.Set("headers", headers) | ||
flumeRecord.Set("body", []byte(msg.Line)) | ||
|
||
logrus.WithFields(logrus.Fields{"flumeRecord": flumeRecord}).Info("Avro logger request") | ||
err := a.requestor.Request("append", flumeRecord) | ||
return err | ||
} | ||
|
||
func (a *avro) Close() error { | ||
return a.conn.Close() | ||
} | ||
|
||
func (a *avro) Name() string { | ||
return name | ||
} | ||
|
||
// ValidateLogOpt looks for avro specific log option avro-host avro-port. | ||
func ValidateLogOpt(cfg map[string]string) error { | ||
for key := range cfg { | ||
switch key { | ||
case "env": | ||
case "labels": | ||
case hostKey: | ||
case portKey: | ||
// Accepted | ||
default: | ||
return fmt.Errorf("unknown log opt '%s' for avro log driver", key) | ||
} | ||
} | ||
if len(cfg[hostKey]) == 0 { | ||
cfg[hostKey] = defaultHost | ||
} | ||
if len(cfg[portKey]) == 0 { | ||
cfg[portKey] = defaultPort | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
<!--[metadata]> | ||
+++ | ||
aliases = ["/engine/reference/logging/flume/"] | ||
title = "Flume logging driver" | ||
description = "Describes how to use the flume logging driver." | ||
keywords = ["Flume, avro, docker, logging, driver"] | ||
[menu.main] | ||
parent = "smn_logging" | ||
+++ | ||
<![end-metadata]--> | ||
|
||
# Flume logging driver | ||
|
||
The `flume-avor` logging driver send container logs to the [Apache Flume](https://flume.apache.org/) collector as structured log data. Then, isers can use any of the [various sink of Flume](https://flume.apache.org/FlumeUserGuide.html#flume-sinks) to write these logs to various destinations; | ||
|
||
In addition to the log message itself, the `flume`log driver sends the following metadata in the structured log message: | ||
|
||
| Field | Description | | ||
-------------------|-------------------------------------| | ||
| `container_id` | The full 64-character container ID. | | ||
| `container_name` | The container name at the time it was started. If you use `docker rename` to rename a container, the new name is not reflected in the journal entries. | | ||
| `source` | `stdout` or `stderr` | | ||
|
||
The `docker logs` command is not available for this logging driver. | ||
|
||
## usage | ||
|
||
Some options are supported by specifying `--log-opt` as many times as needed: | ||
|
||
- `avro-host`: specify `host` to connect `localhost` | ||
- `avro-port`: specify `port` to connect `63001` | ||
|
||
Configure the default logging driver by passing the | ||
`--log-driver` option to the Docker daemon: | ||
|
||
docker daemon --log-driver=flume-avro | ||
|
||
To set the logging driver for a specific container, pass the | ||
`--log-driver` option to `docker run`: | ||
|
||
docker run --log-driver=flume-avro ... | ||
|
||
Before using this logging driver, launch a Flume daemon. The logging driver connects to this daemon through `localhost:63001` by default. Use the `flume-host=localhost` and `flume-port=63001` option to connect to a different address. | ||
|
||
|
||
docker run --log-driver=flume<F4>avro --log-opt flume-host=myhost.local --log-opt flume-port=24224 | ||
|
||
If container cannot connect to the Flume daemon, the container stops immediately. | ||
|
||
## Options | ||
|
||
Users can use the `--log-opt NAME=VALUE` flag to specify additional Flume logging driver options. | ||
|
||
### avro-host | ||
|
||
By default, the logging driver connects to `localhost`. Supply the `avr-host` option to connect to a different address. | ||
|
||
### avro-port | ||
|
||
By default, the logging driver connects to `63001`. Supply the `avr-port` option to connect to a different port. | ||
|
||
|
||
## Flume daemon management with Docker | ||
|
||
About `Apache Flume` itself, see [the project webpage](http://flume.apache.org) and [its documents](http://flume.apache.org/FlumeUserGuide.html). | ||
|
||
To use this logging driver,s tart the `flume` daemon on a host. We recommend that use [the Flume docker image](). This | ||
|
||
## Testing container loggers | ||
|
||
1. Write a configuration file (`flume.conf`) to dump input logs: | ||
|
||
``` | ||
agentLocal.sources = s1 | ||
agentLocal.channels = c1 | ||
agentLocal.sinks = k1 | ||
# Channels | ||
#--------- | ||
agentLocal.channels.c1.type = memory | ||
# Source Avro | ||
#------------ | ||
agentLocal.sources.s1.type=avro | ||
agentLocal.sources.s1.port=63001 | ||
agentLocal.sources.s1.bind=0.0.0.0 | ||
agentLocal.sources.s1.channels=c1 | ||
agentLocal.sources.s1.byteCapacityBufferPercentage=20 | ||
# Configuration de la destination | ||
#-------------------------------- | ||
agentLocal.sinks.k1.type = logger | ||
agentLocal.sinks.k1.channel = c1 | ||
``` | ||
|
||
|
||
2. Launch Flume container with this configuration file: | ||
|
||
$ docker run -it -p 63001:63001 -v /path/to/conf/test.conf:/var/tmp/flume.conf -e FLUME_CONF_FILE=/var/tmp/flume.conf -e FLUME_AGENT_NAME=agentLocal probablyfine/flume:latest | ||
|
||
3. Start one or more containers with the `flume` logging driver: | ||
|
||
$ docker run --log-driver=flume-avro hello-world | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
cmd/snappytool/snappytool | ||
testdata/bench | ||
|
||
# These explicitly listed benchmark data files are for an obsolete version of | ||
# snappy_test.go. | ||
testdata/alice29.txt | ||
testdata/asyoulik.txt | ||
testdata/fireworks.jpeg | ||
testdata/geo.protodata | ||
testdata/html | ||
testdata/html_x_4 | ||
testdata/kppkn.gtb | ||
testdata/lcet10.txt | ||
testdata/paper-100k.pdf | ||
testdata/plrabn12.txt | ||
testdata/urls.10K |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
# This is the official list of Snappy-Go authors for copyright purposes. | ||
# This file is distinct from the CONTRIBUTORS files. | ||
# See the latter for an explanation. | ||
|
||
# Names should be added to this file as | ||
# Name or Organization <email address> | ||
# The email address is not required for organizations. | ||
|
||
# Please keep the list sorted. | ||
|
||
Damian Gryski <dgryski@gmail.com> | ||
Google Inc. | ||
Jan Mercl <0xjnml@gmail.com> | ||
Rodolfo Carvalho <rhcarvalho@gmail.com> | ||
Sebastien Binet <seb.binet@gmail.com> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
# This is the official list of people who can contribute | ||
# (and typically have contributed) code to the Snappy-Go repository. | ||
# The AUTHORS file lists the copyright holders; this file | ||
# lists people. For example, Google employees are listed here | ||
# but not in AUTHORS, because Google holds the copyright. | ||
# | ||
# The submission process automatically checks to make sure | ||
# that people submitting code are listed in this file (by email address). | ||
# | ||
# Names should be added to this file only after verifying that | ||
# the individual or the individual's organization has agreed to | ||
# the appropriate Contributor License Agreement, found here: | ||
# | ||
# http://code.google.com/legal/individual-cla-v1.0.html | ||
# http://code.google.com/legal/corporate-cla-v1.0.html | ||
# | ||
# The agreement for individuals can be filled out on the web. | ||
# | ||
# When adding J Random Contributor's name to this file, | ||
# either J's name or J's organization's name should be | ||
# added to the AUTHORS file, depending on whether the | ||
# individual or corporate CLA was used. | ||
|
||
# Names should be added to this file like so: | ||
# Name <email address> | ||
|
||
# Please keep the list sorted. | ||
|
||
Damian Gryski <dgryski@gmail.com> | ||
Jan Mercl <0xjnml@gmail.com> | ||
Kai Backman <kaib@golang.org> | ||
Marc-Antoine Ruel <maruel@chromium.org> | ||
Nigel Tao <nigeltao@golang.org> | ||
Rob Pike <r@golang.org> | ||
Rodolfo Carvalho <rhcarvalho@gmail.com> | ||
Russ Cox <rsc@golang.org> | ||
Sebastien Binet <seb.binet@gmail.com> |
Oops, something went wrong.