Merge branch 'master' into doublemarket/json-plugin

.travis.yml

@@ -1,6 +1,6 @@
 language: go
 go:
-- 1.8
+- 1.9
 env:
   global:
   - PATH=~/gopath/bin:$PATH DEBIAN_FRONTEND=noninteractive

CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## 0.35.0 (2017-10-04)
+
+* [twemproxy] [incompatible] add `-enable-each-server-metrics` option #419 #421 (Songmu)
+
+
+## 0.34.0 (2017-09-27)
+
+* add mackerel-plugin-flume to package #415 (y-kuno)
+* [mysql]add MyISAM related graphs #406 (matsuu)
+* add mackerel-plugin-sidekiq to package #417 (syou6162)
+* build with Go 1.9 #414 (astj)
+* [OpenLDAP] fix get latestCSN #413 (masahide)
+* [aws-dynamodb] Add ReadThrottleEvents metric and fill 0 when *ThrottleEvents metrics are not present #409 (astj)
+
+
+## 0.33.0 (2017-09-20)
+
+* add mackerel-plugin-nvidia-smi to package #411 (syou6162)
+* [accesslog] Feature/accesslog/customize parser #410 (karupanerura)
+* Fix redundant error by golint in redis.go #408 (shibayu36)
+* add flume plugin #396 (y-kuno)
+* [mysql]add handler graphs #402 (matsuu)
+
+
+## 0.32.0 (2017-09-12)
+
+* [memcached] add evicted.reclaimed and evicted.nonzero_evictions #388 (Songmu)
+* [mysql]add missed metrics and fix graph definition #390 (matsuu)
+* [Redis] fix expired keys #398 (edangelion)
+* [accesslog] Fix for scanning long lines #400 (itchyny)
+
+
+## 0.31.0 (2017-08-30)
+
+* [redis] Change queries metric to diff of "total_commands_processed" #397 (edangelion)
+* [aws-dynamodb] Refactor and parallelize CloudWatch request with errgroup #367 (astj)
+* [plack] Don't raise errors when parsing JSON fields failed #394 (astj)
+* [jmx-jolokia] add value to thread graph #393 (y-kuno)
+
+
 ## 0.30.0 (2017-08-23)
 
 * add mackerel-plugin-openldap to package #391 (astj)

Makefile

@@ -1,4 +1,4 @@
-VERSION = 0.30.0
+VERSION = 0.35.0
 VERBOSE_FLAG = $(if $(VERBOSE),-verbose)
 CURRENT_REVISION = $(shell git rev-parse --short HEAD)
 

README.md

@@ -28,6 +28,7 @@ Documentation for each plugin is located in its respective sub directory.
 * [mackerel-plugin-docker](./mackerel-plugin-docker/README.md)
 * [mackerel-plugin-elasticsearch](./mackerel-plugin-elasticsearch/README.md)
 * [mackerel-plugin-fluentd](./mackerel-plugin-fluentd/README.md)
+* [mackerel-plugin-flume](./mackerel-plugin-flume/README.md)
 * [mackerel-plugin-gcp-compute-engine](./mackerel-plugin-gcp-compute-engine/README.md)
 * [mackerel-plugin-gearmand](./mackerel-plugin-gearmand/README.md)
 * [mackerel-plugin-gostats](./mackerel-plugin-gostats/README.md)

mackerel-plugin-accesslog/lib/accesslog.go

@@ -2,6 +2,7 @@ package mpaccesslog
 
 import (
 	"bufio"
+	"bytes"
 	"flag"
 	"fmt"
 	"io"
@@ -24,6 +25,7 @@ type AccesslogPlugin struct {
 	prefix    string
 	file      string
 	posFile   string
+	parser    axslogparser.Parser
 	noPosFile bool
 }
 
@@ -117,18 +119,33 @@ func (p *AccesslogPlugin) FetchMetrics() (map[string]float64, error) {
 		ret[k] = 0
 	}
 	var reqtimes []float64
-	var psr axslogparser.Parser
-	s := bufio.NewScanner(rc)
-	for s.Scan() {
+	r := bufio.NewReader(rc)
+	for {
 		var (
 			l   *axslogparser.Log
 			err error
+			bb  bytes.Buffer
 		)
-		line := s.Text()
-		if psr == nil {
-			psr, l, err = axslogparser.GuessParser(line)
+		buf, isPrefix, err := r.ReadLine()
+		bb.Write(buf)
+		for isPrefix {
+			buf, isPrefix, err = r.ReadLine()
+			if err != nil {
+				break
+			}
+			bb.Write(buf)
+		}
+		if err != nil {
+			if err != io.EOF {
+				log.Println(err)
+			}
+			break
+		}
+		line := bb.String()
+		if p.parser == nil {
+			p.parser, l, err = axslogparser.GuessParser(line)
 		} else {
-			l, err = psr.Parse(line)
+			l, err = p.parser.Parse(line)
 		}
 		if err != nil {
 			log.Println(err)
@@ -143,9 +160,6 @@ func (p *AccesslogPlugin) FetchMetrics() (map[string]float64, error) {
 			reqtimes = append(reqtimes, *l.TakenSec)
 		}
 	}
-	if s.Err() != nil {
-		log.Println(s.Err())
-	}
 	if ret["total_count"] > 0 {
 		for _, v := range []string{"2xx", "3xx", "4xx", "5xx"} {
 			ret[v+"_percentage"] = ret[v+"_count"] * 100 / ret["total_count"]
@@ -169,6 +183,7 @@ func (p *AccesslogPlugin) FetchMetrics() (map[string]float64, error) {
 func Do() {
 	var (
 		optPrefix    = flag.String("metric-key-prefix", "", "Metric key prefix")
+		optFormat    = flag.String("format", "", "Access Log format ('ltsv' or 'apache')")
 		optPosFile   = flag.String("posfile", "", "(not necessary to specify it in the usual use case) posfile")
 		optNoPosFile = flag.Bool("no-posfile", false, "no position file")
 	)
@@ -181,10 +196,26 @@ func Do() {
 		flag.Usage()
 		os.Exit(1)
 	}
+
+	var parser axslogparser.Parser
+	switch *optFormat {
+	case "":
+		parser = nil // guess format by log (default)
+	case "ltsv":
+		parser = &axslogparser.LTSV{}
+	case "apache":
+		parser = &axslogparser.Apache{}
+	default:
+		fmt.Fprintf(os.Stderr, "Error: '%s' is invalid format name\n", *optFormat)
+		flag.Usage()
+		os.Exit(1)
+	}
+
 	mp.NewMackerelPlugin(&AccesslogPlugin{
 		prefix:    *optPrefix,
 		file:      flag.Args()[0],
 		posFile:   *optPosFile,
 		noPosFile: *optNoPosFile,
+		parser:    parser,
 	}).Run()
 }

mackerel-plugin-accesslog/lib/accesslog_test.go

@@ -3,6 +3,8 @@ package mpaccesslog
 import (
 	"reflect"
 	"testing"
+
+	"github.com/Songmu/axslogparser"
 )
 
 var fetchMetricsTests = []struct {
@@ -44,6 +46,25 @@ var fetchMetricsTests = []struct {
 			"99_percentile":  3.018,
 		},
 	},
+	{
+		Name:   "LTSV long line log",
+		InFile: "testdata/sample-ltsv-long.tsv",
+		Output: map[string]float64{
+			"2xx_count":      2,
+			"3xx_count":      0,
+			"4xx_count":      0,
+			"5xx_count":      0,
+			"total_count":    2,
+			"2xx_percentage": 100,
+			"3xx_percentage": 0,
+			"4xx_percentage": 0,
+			"5xx_percentage": 0,
+			"average":        0.015,
+			"90_percentile":  0.015,
+			"95_percentile":  0.015,
+			"99_percentile":  0.015,
+		},
+	},
 }
 
 func TestFetchMetrics(t *testing.T) {
@@ -63,3 +84,59 @@ func TestFetchMetrics(t *testing.T) {
 		}
 	}
 }
+
+func TestFetchMetricsWithCustomParser(t *testing.T) {
+	// OK case
+	p := &AccesslogPlugin{
+		file:      "testdata/sample-ltsv.tsv",
+		noPosFile: true,
+		parser:    &axslogparser.LTSV{},
+	}
+	out, err := p.FetchMetrics()
+	if err != nil {
+		t.Errorf("error should be nil but: %+v", err)
+		return
+	}
+
+	expected := map[string]float64{
+		"2xx_count":      7,
+		"3xx_count":      1,
+		"4xx_count":      1,
+		"5xx_count":      1,
+		"total_count":    10,
+		"2xx_percentage": 70,
+		"3xx_percentage": 10,
+		"4xx_percentage": 10,
+		"5xx_percentage": 10,
+		"average":        0.7603999999999999,
+		"90_percentile":  2.018,
+		"95_percentile":  3.018,
+		"99_percentile":  3.018,
+	}
+	if !reflect.DeepEqual(out, expected) {
+		t.Errorf("out:  %#v\n want: %#v", out, expected)
+	}
+
+	// NG case (should not detect log format by log line)
+	p = &AccesslogPlugin{
+		file:      "testdata/sample-apache.log",
+		noPosFile: true,
+		parser:    &axslogparser.LTSV{},
+	}
+	out, err = p.FetchMetrics()
+	if err != nil {
+		t.Errorf("error should be nil but: %+v", err)
+		return
+	}
+
+	expected = map[string]float64{
+		"2xx_count":   0,
+		"3xx_count":   0,
+		"4xx_count":   0,
+		"5xx_count":   0,
+		"total_count": 0,
+	}
+	if !reflect.DeepEqual(out, expected) {
+		t.Errorf("out:  %#v\n want: %#v", out, expected)
+	}
+}