From aacdb12a4372a36563ac24c6d20f76c2c7ae28f2 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 7 Sep 2017 16:14:10 +0200 Subject: [PATCH] README --- README.md | 109 ++++++++++++++++++++++++++++++ cmd/ax/query.go | 17 +++-- pkg/backend/common/client.go | 10 ++- pkg/backend/common/client_test.go | 101 +++++++++++++-------------- pkg/backend/kibana/query.go | 44 ++++++++---- 5 files changed, 210 insertions(+), 71 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..035d9fb --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# Ax +It's a structured logging world we live in, but do we really have to look at JSON? Not with Ax. + +## Installation +For now there's no pre-built binaries, so to run this you need a reasonably recent version of Go. Then either git clone this project in `$GOPATH/src/github.com/zefhemel/ax` or run `go get -u github.com/zefhemel/ax`. + +To install dependencies: + + make dep + +To run tests: + + make test + +To "go install" ax (this will put the resulting binary in `$GOPATH/bin` so put that in your `$PATH`) + + make + + +## Upgrade + +In `$GOPATH/src/github.com/zefhemel/ax`: + + git pull + make + +## Setup +Once you have `ax` installed, the first thing you'll want to do is setup bash or zsh command completion (I'm not kidding). + +For bash, add to `~/.bash_profile`: + + eval "$(ax --completion-script-bash)" + +For zsh, add to `~/.zshrc`: + + eval "$(ax --completion-script-zsh)" + +After this, you can auto complete commands, flags, environments and even attribute names with TAB. Use it, love it. + +## Setup with Kibana +To setup Ax for use with Kibana, run: + + ax env add + +This will prompt you for a name, backend-type (kibana in this case), URL and if this URL is basic auth protected a username and password, and then an index. + +To see if it works, just run: + + ax --env yourenvname + +Or, most likely your new env is the default (check with `ax env`) and you can just run: + + ax + +This should show you the (200) most recent logs. + +If you're comfortable with YAML, you can run `ax env edit` which will open an editor with the `~/.config/ax/ax.yaml` file (either the editor set in your `EDITOR` env variable, with a fallback to `nano`). In there you can easily create more environments quickly. + +## Use with Docker +To use Ax with docker, simply use the `--docker` flag and a container name pattern. I usually use auto complete here (which works for docker containers too): + + ax --docker turbo_ + +To query logs for all containers with "turbo\_" in the name. This assumes you have the `docker` binary in your path and setup properly. + +## Use with log files or processes +You can also pipe logs directly into Ax: + + tail -f /var/log/something.log | ax + +# Filtering and selecting attributes +Looking at all logs is nice, but it only gets really interesting if you can start to filter stuff and by selecting only certain attributes. + +To search for all logs containing the phrase "Traceback": + + ax "Traceback" + +To search for all logs with the phrase "Traceback" and where the attribute "domain" is set to "zef": + + ax --where domain=zef "Traceback" + +Again, after running Ax once on an environment it will cache attribute names, so you get completion for those too, usually. + +Ax also supports the `!=` operator: + + ax --where domain!=zef + +If you have a lot of extra attributes in your log messages, you can select just a few of them: + + ax --where domain=zef --select message --select tag + +# "Tailing" logs +Use the `-f` flag: + + ax -f --where domain=zef + +# Different output formats +Don't like the default textual output, perhaps you prefer YAML: + + ax --output yaml + +or pretty JSON: + + ax --output pretty-json + +# Getting help + + ax --help + ax query --help diff --git a/cmd/ax/query.go b/cmd/ax/query.go index dcaaf6c..d79cc4b 100644 --- a/cmd/ax/query.go +++ b/cmd/ax/query.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" "time" @@ -57,17 +58,21 @@ func selectHintAction() []string { return resultList } +var filterRegex = regexp.MustCompile(`([^!=<>]+)\s*(=|!=)\s*(.*)`) + func buildFilters(wheres []string) []common.QueryFilter { filters := make([]common.QueryFilter, 0, len(wheres)) for _, whereClause := range wheres { - pieces := strings.SplitN(whereClause, "=", 2) - if len(pieces) != 2 { + //pieces := strings.SplitN(whereClause, "=", 2) + matches := filterRegex.FindAllStringSubmatch(whereClause, -1) + if len(matches) != 1 { fmt.Println("Invalid where clause", whereClause) os.Exit(1) } filters = append(filters, common.QueryFilter{ - FieldName: pieces[0], - Value: pieces[1], + FieldName: matches[0][1], + Operator: matches[0][2], + Value: matches[0][3], }) } return filters @@ -126,10 +131,10 @@ func printMessage(message common.LogMessage, queryOutputFormat string) { fmt.Printf("%s ", messageColor.Sprint(msg)) } for key, value := range message.Attributes { - if key == "message" { + if key == "message" || value == nil { continue } - fmt.Printf("%s=%s ", color.CyanString(key), common.MustJsonEncode(value)) + fmt.Printf("%s=%+v ", color.CyanString(key), value) } fmt.Println() case "json": diff --git a/pkg/backend/common/client.go b/pkg/backend/common/client.go index 98b5b2c..5f9ed71 100644 --- a/pkg/backend/common/client.go +++ b/pkg/backend/common/client.go @@ -16,6 +16,7 @@ type Client interface { type QueryFilter struct { FieldName string + Operator string Value string } @@ -126,7 +127,14 @@ func Project(m map[string]interface{}, fields []string) map[string]interface{} { func (f QueryFilter) Matches(m LogMessage) bool { val, ok := m.Attributes[f.FieldName] - return ok && f.Value == fmt.Sprintf("%v", val) + switch f.Operator { + case "=": + return ok && f.Value == fmt.Sprintf("%v", val) + case "!=": + return !ok || (ok && f.Value != fmt.Sprintf("%v", val)) + default: + panic("Not supported operator") + } } func matchesPhrase(s, phrase string) bool { diff --git a/pkg/backend/common/client_test.go b/pkg/backend/common/client_test.go index b63bc15..93997c6 100644 --- a/pkg/backend/common/client_test.go +++ b/pkg/backend/common/client_test.go @@ -16,64 +16,67 @@ func TestFilter(t *testing.T) { } lastHour := time.Now().Add(-time.Hour) nextHour := time.Now().Add(time.Hour) - shouldMatchQuery := Query{ - Filters: []QueryFilter{ - QueryFilter{FieldName: "someStr", Value: "Zef"}, + shouldMatchQueries := []Query{ + Query{ + Filters: []QueryFilter{ + QueryFilter{FieldName: "someStr", Value: "Zef", Operator: "="}, + }, }, - } - shouldMatchQuery2 := Query{ - Filters: []QueryFilter{ - QueryFilter{FieldName: "someN", Value: "34"}, + Query{ + Filters: []QueryFilter{ + QueryFilter{FieldName: "someN", Value: "34", Operator: "="}, + }, }, - } - shouldMatchQuery3 := Query{ - QueryString: "zef", - Filters: []QueryFilter{ - QueryFilter{FieldName: "someN", Value: "34"}, + Query{ + QueryString: "zef", + Filters: []QueryFilter{ + QueryFilter{FieldName: "someN", Value: "34", Operator: "="}, + }, }, - } - shouldMatchQuery4 := Query{ - QueryString: "zef", - Filters: []QueryFilter{ - QueryFilter{FieldName: "someN", Value: "34"}, + Query{ + QueryString: "zef", + Filters: []QueryFilter{ + QueryFilter{FieldName: "someN", Value: "34", Operator: "="}, + }, + Before: &nextHour, + After: &lastHour, }, - Before: &nextHour, - After: &lastHour, - } - shouldNotMatchQuery := Query{ - Filters: []QueryFilter{ - QueryFilter{FieldName: "someStr", Value: "Pete"}, + Query{ + Filters: []QueryFilter{ + QueryFilter{FieldName: "someN", Value: "32", Operator: "!="}, + }, }, - } - shouldNotMatchQuery2 := Query{ - QueryString: "bla", - Filters: []QueryFilter{ - QueryFilter{FieldName: "someStr", Value: "Pete"}, + Query{ + Filters: []QueryFilter{ + QueryFilter{FieldName: "someNonexistingField", Value: "Pete", Operator: "!="}, + }, }, } - shouldNotMatchQuery3 := Query{ - After: &nextHour, - } - if !MatchesQuery(lm, shouldMatchQuery) { - t.Errorf("Did not match") - } - if !MatchesQuery(lm, shouldMatchQuery2) { - t.Errorf("Did not match 2") - } - if !MatchesQuery(lm, shouldMatchQuery3) { - t.Errorf("Did not match 3") - } - if !MatchesQuery(lm, shouldMatchQuery4) { - t.Errorf("Did not match 4") - } - if MatchesQuery(lm, shouldNotMatchQuery) { - t.Errorf("Did match") + shouldNotMatchQueries := []Query{ + Query{ + Filters: []QueryFilter{ + QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="}, + }, + }, + Query{ + QueryString: "bla", + Filters: []QueryFilter{ + QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="}, + }, + }, + Query{ + After: &nextHour, + }, } - if MatchesQuery(lm, shouldNotMatchQuery2) { - t.Errorf("Did match 2") + for i, shouldMatch := range shouldMatchQueries { + if !MatchesQuery(lm, shouldMatch) { + t.Errorf("Did not match: %d: %+v", i, shouldMatch) + } } - if MatchesQuery(lm, shouldNotMatchQuery3) { - t.Errorf("Did match 3") + for i, shouldNotMatch := range shouldNotMatchQueries { + if MatchesQuery(lm, shouldNotMatch) { + t.Errorf("Did match: %d: %+v", i, shouldNotMatch) + } } } diff --git a/pkg/backend/kibana/query.go b/pkg/backend/kibana/query.go index 6902410..92ed85a 100644 --- a/pkg/backend/kibana/query.go +++ b/pkg/backend/kibana/query.go @@ -42,7 +42,7 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit, if query.QueryString == "" { queryString = "*" } - filterList := JsonList{ + mustFilters := JsonList{ JsonObject{ "query_string": JsonObject{ "analyze_wildcard": true, @@ -65,17 +65,29 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit, if query.Before != nil { rangeObj["range"].(JsonObject)["@timestamp"].(JsonObject)["lt"] = unixMillis(*query.Before) } - filterList = append(filterList, rangeObj) + mustFilters = append(mustFilters, rangeObj) } + mustNotFilters := JsonList{} for _, filter := range query.Filters { m := JsonObject{} - m[filter.FieldName] = JsonObject{ - "query": filter.Value, - "type": "phrase", + switch filter.Operator { + case "=": + m[filter.FieldName] = JsonObject{ + "query": filter.Value, + "type": "phrase", + } + mustFilters = append(mustFilters, JsonObject{ + "match": m, + }) + case "!=": + m[filter.FieldName] = JsonObject{ + "query": filter.Value, + "type": "phrase", + } + mustNotFilters = append(mustNotFilters, JsonObject{ + "match": m, + }) } - filterList = append(filterList, JsonObject{ - "match": m, - }) } body, err := createMultiSearch( JsonObject{ @@ -94,7 +106,8 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit, }, "query": JsonObject{ "bool": JsonObject{ - "must": filterList, + "must": mustFilters, + "must_not": mustNotFilters, }, }, }) @@ -197,19 +210,20 @@ func (client *Client) querySubIndex(subIndex string, q common.Query) ([]common.L allMessages := make([]common.LogMessage, 0, 200) for _, hit := range hits { - var attributes map[string]interface{} - var ts time.Time - attributes = common.Project(hit.Source, q.SelectFields) - ts, err = time.Parse(time.RFC3339, hit.Source["@timestamp"].(string)) + //var ts time.Time + attributes := hit.Source + ts, err := time.Parse(time.RFC3339, attributes["@timestamp"].(string)) if err != nil { return nil, err } delete(attributes, "@timestamp") - allMessages = append(allMessages, common.FlattenLogMessage(common.LogMessage{ + message := common.FlattenLogMessage(common.LogMessage{ ID: hit.ID, Timestamp: ts, Attributes: attributes, - })) + }) + message.Attributes = common.Project(message.Attributes, q.SelectFields) + allMessages = append(allMessages, message) } return allMessages, nil }