Logstash like, written in golang
Branch: master
Clone or download
tsaikd Merge pull request #78 from tengattack/patch/logevent-pathvalue-slice…
…access

Update logevent get value support slice access
Latest commit 891ecbf Feb 14, 2019
Permalink
Type Name Latest commit message Commit time
Failed to load latest commit information.
cmd Update worker config name Jan 10, 2019
codec/json
config Update logevent get value support slice access Feb 14, 2019
docker 0.1.17 Oct 17, 2018
filter output/http: add output http Jan 29, 2019
input input/beats: update decoder Jan 27, 2019
modloader output/http: add output http Jan 29, 2019
output output/http: add output http Jan 29, 2019
.gitignore Add useragent & geoip2 cache Jan 26, 2019
.travis.yml remove go 1.9 1.10 from travis Jan 3, 2019
Gopkg.lock update dep libs Jul 27, 2018
Gopkg.toml #39 use dep to manage dependencies instead of Godeps Jul 4, 2018
LICENSE Initial commit Dec 10, 2014
README.md Merge pull request #75 from tengattack/patch/filters Jan 27, 2019
main.go
version.go Add worker module Jan 10, 2019

README.md

gogstash

Logstash like, written in golang

Build Status

curl 'https://github.com/tsaikd/gogstash/releases/download/0.1.8/gogstash-Linux-x86_64' -SLo gogstash && chmod +x gogstash
  • Configure for ubuntu-sys.json (example)
{
	"input": [
		{
			"type": "exec",
			"command": "sh",
			"interval": 60,
			"message_prefix": "%{@timestamp} [df] ",
			"args": ["-c", "df -B 1 / | sed 1d"]
		},
		{
			"type": "exec",
			"command": "sh",
			"interval": 60,
			"message_prefix": "%{@timestamp} [diskstat] ",
			"args": ["-c", "grep '0 [sv]da ' /proc/diskstats"]
		},
		{
			"type": "exec",
			"command": "sh",
			"interval": 60,
			"message_prefix": "%{@timestamp} [loadavg] ",
			"args": ["-c", "cat /proc/loadavg"]
		},
		{
			"type": "exec",
			"command": "sh",
			"interval": 60,
			"message_prefix": "%{@timestamp} [netdev] ",
			"args": ["-c", "grep '\\beth0:' /proc/net/dev"]
		},
		{
			"type": "exec",
			"command": "sh",
			"interval": 60,
			"message_prefix": "%{@timestamp} [meminfo]\n",
			"args": ["-c", "cat /proc/meminfo"]
		}
	],
	"output": [
		{
			"type": "report"
		},
		{
			"type": "redis",
			"key": "gogstash-ubuntu-sys-%{host}",
			"host": ["127.0.0.1:6379"]
		}
	]
}
  • Configure for dockerstats.json (example)
{
	"input": [
		{
			"type": "dockerstats"
		}
	],
	"output": [
		{
			"type": "report"
		},
		{
			"type": "redis",
			"key": "gogstash-docker-%{host}",
			"host": ["127.0.0.1:6379"]
		}
	]
}
  • Config format with YAML for dockerstats.json (example)
input:
  - type: dockerstats
output:
  - type: report
  - type: redis
    key: "gogstash-docker-%{host}"
    host:
      - "127.0.0.1:6379"
  • Configure for nginx.yml with gonx filter (example)
chsize: 1000
worker: 2

input:
  - type: redis
    host: redis.server:6379
    key:  filebeat-nginx
    connections: 1

filter:
  - type: gonx
    format: '$clientip - $auth [$time_local] "$full_request" $response $bytes "$referer" "$agent"'
    source: message
  - type: gonx
    format: '$verb $request HTTP/$httpversion'
    source: full_request
  - type: date
    format: ["02/Jan/2006:15:04:05 -0700"]
    source: time_local
  - type: remove_field
    fields: ["full_request", "time_local"]
  - type: add_field
    key: host
    value: "%{beat.hostname}"
  - type: geoip2
    db_path: "GeoLite2-City.mmdb"
    ip_field: clientip
    key: req_geo
  - type: typeconv
    conv_type: int64
    fields: ["bytes", "response"]

output:
  - type: elastic
    url: "http://elastic.server:9200"
    index: "log-nginx-%{+@2006-01-02}"
    document_type: "%{type}"
  • Configure for beats.yml with grok filter (example)
chsize: 1000
worker: 2
event:
  sort_map_keys: false
  remove_field: ['@metadata']

input:
  - type: beats
    port: 5044
    reuseport: true
    host: 0.0.0.0
    ssl:  false

filter:
  - type: grok
    match: ["%{COMMONAPACHELOG}"]
    source: "message"
    patterns_path: "/etc/gogstash/grok-patterns"
  - type: date
    format: ["02/Jan/2006:15:04:05 -0700"]
    source: time_local
  - type: remove_field
    fields: ["full_request", "time_local"]
  - type: add_field
    key: host
    value: "%{beat.hostname}"
  - type: geoip2
    db_path: "GeoLite2-City.mmdb"
    ip_field: clientip
    key: req_geo
  - type: typeconv
    conv_type: int64
    fields: ["bytes", "response"]

output:
  - type: elastic
    url: ["http://elastic1:9200","http://elastic2:9200","http://elastic3:9200"]
    index: "filebeat-6.4.2-%{+@2006.01.02}"
    document_type: "doc"
  • Run gogstash for nginx example (command line)
GOMAXPROCS=4 ./gogstash --CONFIG nginx.json
  • Run gogstash for dockerstats example (docker image)
docker run -it --rm \
	--name gogstash \
	--hostname gogstash \
	-e GOMAXPROCS=4 \
	-v "/var/run/docker.sock:/var/run/docker.sock" \
	-v "${PWD}/dockerstats.json:/gogstash/config.json:ro" \
	tsaikd/gogstash:0.1.8

Supported inputs

See input modules for more information

Supported filters

See filter modules for more information

Supported outputs

See output modules for more information