From dd52580e4659382313d946b09bc42eee07da98d5 Mon Sep 17 00:00:00 2001 From: Peter Lisy Date: Mon, 27 Jun 2022 17:05:03 -0700 Subject: [PATCH] feat: Added logfmt CLI (#2) --- cmd/logfmt/logfmt.go | 18 +++- cmd/logfmt/logfmt_test.go | 82 ++++++++++++++++++ go.mod | 16 +++- go.sum | 28 +++++- internal/runner/runner.go | 177 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 cmd/logfmt/logfmt_test.go create mode 100644 internal/runner/runner.go diff --git a/cmd/logfmt/logfmt.go b/cmd/logfmt/logfmt.go index 3bb08f1..c353201 100644 --- a/cmd/logfmt/logfmt.go +++ b/cmd/logfmt/logfmt.go @@ -13,8 +13,10 @@ import ( gcli "github.com/getoutreach/gobox/pkg/cli" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + // Place any extra imports for your startup code here ///Block(imports) + "github.com/getoutreach/logfmt/internal/runner" ///EndBlock(imports) ) @@ -50,12 +52,24 @@ func main() { Version: oapp.Version, Name: "logfmt", ///Block(app) - + Usage: `make test | logfmt -filter -format `, + Action: func(c *cli.Context) error { + r := runner.New(log, c.String("filter"), c.String("format")) + r.Run() + return nil + }, ///EndBlock(app) } app.Flags = []cli.Flag{ ///Block(flags) - + &cli.StringFlag{ + Name: "filter", + Usage: "filter the log. Use jq syntax", + }, + &cli.StringFlag{ + Name: "format", + Usage: "format the output. Use golang templates syntax", + }, ///EndBlock(flags) } app.Commands = []*cli.Command{ diff --git a/cmd/logfmt/logfmt_test.go b/cmd/logfmt/logfmt_test.go new file mode 100644 index 0000000..7144424 --- /dev/null +++ b/cmd/logfmt/logfmt_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +package main_test + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "gotest.tools/v3/icmd" +) + +type line struct { + Level string `json:"level"` + Message string `json:"message"` + Timestamp string `json:"@timestamp"` +} + +func TestLogfmtNoJSONText(t *testing.T) { + txt := "Line 1.\nLine 2.\n" + + runLogfmt(t, txt, txt) +} + +func TestLogfmtStructured(t *testing.T) { + var buff bytes.Buffer + enc := json.NewEncoder(&buff) + + enc.Encode(line{ + Level: "info", + Message: "Hello, World!", + Timestamp: (time.Time{}.Add(1 * time.Second)).Format(time.RFC3339Nano), + }) + + runLogfmt(t, + buff.String(), + `time="0001-01-01T00:00:01Z" level=info msg="Hello, World!"`) +} + +func TestLogfmtStructuredFormat(t *testing.T) { + var buff bytes.Buffer + enc := json.NewEncoder(&buff) + + enc.Encode(line{Level: "info", Message: "msg1"}) + enc.Encode(line{Level: "info", Message: "msg2"}) + + runLogfmt(t, + buff.String(), + "msg1\nmsg2\n", + "--format", "{{ .message }}", + ) +} + +func TestLogfmtStructuredFilter(t *testing.T) { + var buff bytes.Buffer + enc := json.NewEncoder(&buff) + + enc.Encode(line{Level: "info", Message: "info"}) + enc.Encode(line{Level: "warn", Message: "warn"}) + enc.Encode(line{Level: "error", Message: "error"}) + + runLogfmt(t, + buff.String(), + "info\nerror\n", + "--format", "{{ .message }}", + "--filter", `select(.level != "warn")`, + ) +} + +func runLogfmt(t *testing.T, input, output string, args ...string) { + logs := strings.NewReader(input) + + args = append([]string{"run", `logfmt.go`}, args...) + + result := icmd.RunCmd(icmd.Command("go", args...), icmd.WithStdin(logs)) + result.Assert(t, icmd.Expected{ + ExitCode: 0, + Err: output, + }) +} diff --git a/go.mod b/go.mod index f886f91..33fd0ac 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,17 @@ module github.com/getoutreach/logfmt go 1.17 require ( + github.com/Masterminds/sprig/v3 v3.2.2 github.com/getoutreach/gobox v1.41.5 + github.com/itchyny/gojq v0.12.8 github.com/sirupsen/logrus v1.8.1 github.com/urfave/cli/v2 v2.6.0 + gotest.tools/v3 v3.1.0 ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -24,12 +29,17 @@ require ( github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-github/v43 v43.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/honeycombio/beeline-go v1.4.1 // indirect github.com/honeycombio/libhoney-go v1.15.8 // indirect + github.com/huandu/xstrings v1.3.1 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/klauspost/compress v1.15.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -40,6 +50,8 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/microcosm-cc/bluemonday v1.0.17 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.9.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect @@ -51,6 +63,8 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/schollz/progressbar/v3 v3.8.6 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -59,7 +73,7 @@ require ( golang.org/x/crypto v0.0.0-20220408190544-5352b0902921 // indirect golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect - golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 4241e62..310d3f8 100644 --- a/go.sum +++ b/go.sum @@ -41,7 +41,13 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= +github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -277,7 +283,9 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -301,11 +309,19 @@ github.com/honeycombio/beeline-go v1.4.1 h1:hbw9FVT7AfKicy+VD7U7ZXr8jDa0OBQKGMAB github.com/honeycombio/beeline-go v1.4.1/go.mod h1:fJJO3XznsA3Co/gUIjsCnuTB3TKlezs0ib796SX6ixE= github.com/honeycombio/libhoney-go v1.15.8 h1:TECEltZ48K6J4NG1JVYqmi0vCJNnHYooFor83fgKesA= github.com/honeycombio/libhoney-go v1.15.8/go.mod h1:+tnL2etFnJmVx30yqmoUkVyQjp7uRJw0a2QGu48lSyY= +github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= +github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/itchyny/gojq v0.12.8 h1:Zxcwq8w4IeR8JJYEtoG2MWJZUv0RGY6QqJcO1cqV8+A= +github.com/itchyny/gojq v0.12.8/go.mod h1:gE2kZ9fVRU0+JAksaTzjIlgnCa2akU+a1V0WXgJQN5c= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -434,8 +450,12 @@ github.com/microcosm-cc/bluemonday v1.0.17 h1:Z1a//hgsQ4yjC+8zEkV8IWySkXnsxmdSY6 github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -509,6 +529,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -524,6 +546,8 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -601,6 +625,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -765,8 +790,9 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 0000000..8114f8b --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,177 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +// Description: Package runner implements the bulk of logfmt functionality. + +// Package runner implements the bulk of logfmt functionality. +package runner + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "strings" + "text/template" + "time" + + "github.com/Masterminds/sprig/v3" + "github.com/itchyny/gojq" + "github.com/sirupsen/logrus" +) + +// traceEvent is trace +const traceEvent = "trace" + +// New returns a new runner +func New(log *logrus.Logger, filter, format string) *Runner { + r := &Runner{log: log} + r.filter = r.filterf(filter) + r.format = r.formatf(format) + r.formatStr = format + return r +} + +// Runner is a logger that emits to datadog and formats +// to logrus +type Runner struct { + log *logrus.Logger + filter func(v interface{}) bool + format func(v interface{}) string + + formatStr string +} + +// logrus converts data set to the runner to the logrus +// format. +func (r *Runner) logrus(data map[string]interface{}) { + logrusLog := logrus.NewEntry(r.log) + + level := "" + message := "" + logType := "" + + if v, ok := data["event_name"]; ok { + if v.(string) == traceEvent { + logType = traceEvent + } + } + + for k, v := range data { + // skip empty keys + if sv, ok := v.(string); ok && sv == "" { + continue + } + + switch k { + case "level": + level = v.(string) + continue + case "message": + if logType == traceEvent { + message = "(trace) " + } + message += v.(string) + continue + case "@timestamp": + if t, err := time.Parse(time.RFC3339Nano, v.(string)); err == nil { + logrusLog = logrusLog.WithTime(t) + } + continue + case "app.version", "deployment.namespace", "app.name", "timing.service_time", "timing.dequeued_at", + "timing.finished_at", "timing.scheduled_at", "timing.total_time", "timing.wait_time", "honeycomb.trace_id", + "event_name": + continue + } + logrusLog = logrusLog.WithField(k, v) + } + + switch strings.ToLower(level) { + case "info": + logrusLog.Info(message) + case "warn": + logrusLog.Warn(message) + case "error", "fatal": + logrusLog.Error(message) + } +} + +// Run starts the runner +func (r *Runner) Run() { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + text := scanner.Text() + var data map[string]interface{} + err := json.Unmarshal([]byte(text), &data) + if err != nil { + fmt.Fprintln(r.log.Out, text) + continue + } + + if r.filter(data) { + if r.formatStr != "" { + fmt.Fprintln(r.log.Out, r.format(data)) + } else { + r.logrus(data) + } + } + } +} + +// filterf filters out logs +func (r *Runner) filterf(s string) func(interface{}) bool { + if s == "" { + return func(_ interface{}) bool { + return true + } + } + + query, err := gojq.Parse(s) + r.must(err) + + code, err := gojq.Compile(query) + r.must(err) + + return func(data interface{}) bool { + result, ok := code.Run(data).Next() + if !ok { + return false + } + if err, ok := result.(error); ok { + r.must(err) + } + return true + } +} + +// formatf formats the logs with the given format string +func (r *Runner) formatf(s string) func(interface{}) string { + jsonFormat := func(v interface{}) string { + if text, ok := v.(string); ok { + return text + } + data, err := json.Marshal(v) + r.must(err) + return string(data) + } + + if s == "" { + return jsonFormat + } + + t := template.Must(template.New("format").Funcs(sprig.TxtFuncMap()).Parse(s)) + return func(v interface{}) string { + var buf bytes.Buffer + if err := t.Execute(&buf, v); err != nil { + r.log.Errorf("format template error %v", err) + } + return buf.String() + } +} + +// must is a handler for panics +func (r *Runner) must(err error) { + if err != nil { + r.log.Fatal("error ", err) + } +}