initial commit

.gitignore

@@ -0,0 +1,7 @@
+/.idea
+/build
+
+.DS_Store
+/varnishlogbeat
+/varnishlogbeat.test
+*.pyc

LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2016 Jimena Cabrera Notari
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,45 @@
+# varnishlogbeat
+
+varnishlogbeat collects log data from a Varnish Shared Memory file and ships it
+to Elasticseach.
+
+varnishlogbeat uses [vago](phenomenes/vago).
+
+## Status
+
+varnishlogbeat is currently in beta but is functional. If you encounter permormance
+issues or bugs, please create an issue or send a pull request.
+
+
+### Requirements
+
+* [Golang](https://golang.org/dl/) 1.7
+* pkg-config
+* [varnish-dev](http://www.varnish-cache.org/releases/) >= 4.1
+
+You will also need to set `PKG_CONFIG_PATH` to the directory where
+`varnishapi.pc` is located before running `go get`. For example:
+
+```
+export PKG_CONFIG_PATH=/usr/lib/pkgconfig
+```
+
+### Build
+
+```
+go get github.com/phenomenes/varnishlogbeat
+cd $GOPATH/src/github.com/phenomenes/varnishlogbeat
+go build .
+```
+
+### Run
+
+Install and run [elasticsearch](elastic/elasticsearch).
+
+Run `varnishlogbeat` with debugging output enabled:
+
+```
+./varnishlogbeat -c varnishlogbeat.yml -e -d "*"
+```
+
+Additionally you can install [kibana](elastic/kibana) to visualize the data.

beater/varnishlogbeat.go

@@ -0,0 +1,112 @@
+package beater
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/publisher"
+
+	"github.com/phenomenes/vago"
+	"github.com/phenomenes/varnishlogbeat/config"
+)
+
+type Varnishlogbeat struct {
+	done    chan struct{}
+	config  config.Config
+	client  publisher.Client
+	varnish *vago.Varnish
+}
+
+// New creates beater
+func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
+	config := config.DefaultConfig
+	if err := cfg.Unpack(&config); err != nil {
+		return nil, fmt.Errorf("Error reading config file: %v", err)
+	}
+
+	vb := Varnishlogbeat{
+		//done:   make(chan struct{}),
+		config: config,
+	}
+
+	return &vb, nil
+}
+
+func (vb *Varnishlogbeat) Run(b *beat.Beat) error {
+	var err error
+
+	logp.Info("varnishlogbeat is running! Hit CTRL-C to stop it.")
+
+	vb.varnish, err = vago.Open(vb.config.Path)
+	if err != nil {
+
+		return err
+	}
+
+	vb.client = b.Publisher.Connect()
+	err = vb.harvest()
+	if err != nil {
+		logp.Err("%s", err)
+	}
+
+	return err
+}
+
+func (vb *Varnishlogbeat) harvest() error {
+	tx := make(common.MapStr)
+	counter := 1
+
+	vb.varnish.Log("", vago.REQ, func(vxid uint32, tag, _type, data string) int {
+		switch _type {
+		case "c":
+			_type = "client"
+		case "b":
+			_type = "backend"
+		default:
+			return 0
+		}
+
+		switch tag {
+		case "BereqHeader", "BerespHeader", "ObjHeader", "ReqHeader", "RespHeader":
+			header := strings.SplitN(data, ": ", 2)
+			k := header[0]
+			v := header[1]
+			if _, ok := tx[tag]; ok {
+				tx[tag].(common.MapStr)[k] = v
+			} else {
+				tx[tag] = common.MapStr{k: v}
+			}
+		case "End":
+			event := common.MapStr{
+				"@timestamp": common.Time(time.Now()),
+				"count":      counter,
+				"type":       _type,
+				"vxid":       vxid,
+				"tx":         tx,
+			}
+			vb.client.PublishEvent(event)
+			counter++
+			logp.Info("Event sent")
+
+			// destroy and re-create the map
+			tx = nil
+			tx = make(common.MapStr)
+		default:
+			tx[tag] = data
+		}
+
+		return 0
+	})
+
+	return nil
+}
+
+func (vb *Varnishlogbeat) Stop() {
+	vb.varnish.Stop()
+	vb.varnish.Close()
+	vb.client.Close()
+}

config/config.go

@@ -0,0 +1,12 @@
+// Config is put into a different package to prevent cyclic imports in case
+// it is needed in several locations
+
+package config
+
+type Config struct {
+	Path string `config:"path"`
+}
+
+var DefaultConfig = Config{
+	Path: "",
+}

config/config_test.go

@@ -0,0 +1,3 @@
+// +build !integration
+
+package config

glide.yaml

@@ -0,0 +1,10 @@
+package: github.com/phenomenes/varnishlogbeat
+import:
+- package: github.com/elastic/beats
+  version: ^5.2.0
+  subpackages:
+  - libbeat/beat
+  - libbeat/common
+  - libbeat/logp
+  - libbeat/publisher
+- package: github.com/phenomenes/vago

main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"os"
+
+	"github.com/elastic/beats/libbeat/beat"
+
+	"github.com/phenomenes/varnishlogbeat/beater"
+)
+
+func main() {
+	err := beat.Run("varnishlogbeat", "", beater.New)
+	if err != nil {
+		os.Exit(1)
+	}
+}