README

README.md

@@ -0,0 +1,109 @@
+# Ax
+It's a structured logging world we live in, but do we really have to look at JSON? Not with Ax.
+
+## Installation
+For now there's no pre-built binaries, so to run this you need a reasonably recent version of Go. Then either git clone this project in `$GOPATH/src/github.com/zefhemel/ax` or run `go get -u github.com/zefhemel/ax`.
+
+To install dependencies:
+
+    make dep
+
+To run tests:
+
+    make test
+
+To "go install" ax (this will put the resulting binary in `$GOPATH/bin` so put that in your `$PATH`)
+
+    make
+
+
+## Upgrade
+
+In `$GOPATH/src/github.com/zefhemel/ax`:
+
+    git pull
+    make
+
+## Setup
+Once you have `ax` installed, the first thing you'll want to do is setup bash or zsh command completion (I'm not kidding).
+
+For bash, add to `~/.bash_profile`:
+
+    eval "$(ax --completion-script-bash)"
+
+For zsh, add to `~/.zshrc`:
+
+    eval "$(ax --completion-script-zsh)"
+
+After this, you can auto complete commands, flags, environments and even attribute names  with TAB. Use it, love it.
+
+## Setup with Kibana
+To setup Ax for use with Kibana, run:
+
+    ax env add
+
+This will prompt you for a name, backend-type (kibana in this case), URL and if this URL is basic auth protected a username and password, and then an index.
+
+To see if it works, just run:
+
+    ax --env yourenvname
+
+Or, most likely your new env is the default (check with `ax env`) and you can just run:
+
+    ax
+
+This should show you the (200) most recent logs.
+
+If you're comfortable with YAML, you can run `ax env edit` which will open an editor with the `~/.config/ax/ax.yaml` file (either the editor set in your `EDITOR` env variable, with a fallback to `nano`). In there you can easily create more environments quickly.
+
+## Use with Docker
+To use Ax with docker, simply use the `--docker` flag and a container name pattern. I usually use auto complete here (which works for docker containers too):
+
+    ax --docker turbo_
+
+To query logs for all containers with "turbo\_" in the name. This assumes you have the `docker` binary in your path and setup properly.
+
+## Use with log files or processes
+You can also pipe logs directly into Ax:
+
+    tail -f /var/log/something.log | ax
+
+# Filtering and selecting attributes
+Looking at all logs is nice, but it only gets really interesting if you can start to filter stuff and by selecting only certain attributes.
+
+To search for all logs containing the phrase "Traceback":
+
+    ax "Traceback"
+
+To search for all logs with the phrase "Traceback" and where the attribute "domain" is set to "zef":
+
+    ax --where domain=zef "Traceback"
+
+Again, after running Ax once on an environment it will cache attribute names, so you get completion for those too, usually.
+
+Ax also supports the `!=` operator:
+
+    ax --where domain!=zef
+
+If you have a lot of extra attributes in your log messages, you can select just a few of them:
+
+    ax --where domain=zef --select message --select tag
+
+# "Tailing" logs
+Use the `-f` flag:
+
+    ax -f --where domain=zef
+
+# Different output formats
+Don't like the default textual output, perhaps you prefer YAML:
+
+    ax --output yaml
+
+or pretty JSON:
+
+    ax --output pretty-json
+
+# Getting help
+
+    ax --help
+    ax query --help

cmd/ax/query.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -57,17 +58,21 @@ func selectHintAction() []string {
 	return resultList
 }
 
+var filterRegex = regexp.MustCompile(`([^!=<>]+)\s*(=|!=)\s*(.*)`)
+
 func buildFilters(wheres []string) []common.QueryFilter {
 	filters := make([]common.QueryFilter, 0, len(wheres))
 	for _, whereClause := range wheres {
-		pieces := strings.SplitN(whereClause, "=", 2)
-		if len(pieces) != 2 {
+		//pieces := strings.SplitN(whereClause, "=", 2)
+		matches := filterRegex.FindAllStringSubmatch(whereClause, -1)
+		if len(matches) != 1 {
 			fmt.Println("Invalid where clause", whereClause)
 			os.Exit(1)
 		}
 		filters = append(filters, common.QueryFilter{
-			FieldName: pieces[0],
-			Value:     pieces[1],
+			FieldName: matches[0][1],
+			Operator:  matches[0][2],
+			Value:     matches[0][3],
 		})
 	}
 	return filters
@@ -126,10 +131,10 @@ func printMessage(message common.LogMessage, queryOutputFormat string) {
 			fmt.Printf("%s ", messageColor.Sprint(msg))
 		}
 		for key, value := range message.Attributes {
-			if key == "message" {
+			if key == "message" || value == nil {
 				continue
 			}
-			fmt.Printf("%s=%s ", color.CyanString(key), common.MustJsonEncode(value))
+			fmt.Printf("%s=%+v ", color.CyanString(key), value)
 		}
 		fmt.Println()
 	case "json":

pkg/backend/common/client.go

@@ -16,6 +16,7 @@ type Client interface {
 
 type QueryFilter struct {
 	FieldName string
+	Operator  string
 	Value     string
 }
 
@@ -126,7 +127,14 @@ func Project(m map[string]interface{}, fields []string) map[string]interface{} {
 
 func (f QueryFilter) Matches(m LogMessage) bool {
 	val, ok := m.Attributes[f.FieldName]
-	return ok && f.Value == fmt.Sprintf("%v", val)
+	switch f.Operator {
+	case "=":
+		return ok && f.Value == fmt.Sprintf("%v", val)
+	case "!=":
+		return !ok || (ok && f.Value != fmt.Sprintf("%v", val))
+	default:
+		panic("Not supported operator")
+	}
 }
 
 func matchesPhrase(s, phrase string) bool {

pkg/backend/common/client_test.go

@@ -16,64 +16,67 @@ func TestFilter(t *testing.T) {
 	}
 	lastHour := time.Now().Add(-time.Hour)
 	nextHour := time.Now().Add(time.Hour)
-	shouldMatchQuery := Query{
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someStr", Value: "Zef"},
+	shouldMatchQueries := []Query{
+		Query{
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someStr", Value: "Zef", Operator: "="},
+			},
 		},
-	}
-	shouldMatchQuery2 := Query{
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someN", Value: "34"},
+		Query{
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
+			},
 		},
-	}
-	shouldMatchQuery3 := Query{
-		QueryString: "zef",
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someN", Value: "34"},
+		Query{
+			QueryString: "zef",
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
+			},
 		},
-	}
-	shouldMatchQuery4 := Query{
-		QueryString: "zef",
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someN", Value: "34"},
+		Query{
+			QueryString: "zef",
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
+			},
+			Before: &nextHour,
+			After:  &lastHour,
 		},
-		Before: &nextHour,
-		After:  &lastHour,
-	}
-	shouldNotMatchQuery := Query{
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someStr", Value: "Pete"},
+		Query{
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someN", Value: "32", Operator: "!="},
+			},
 		},
-	}
-	shouldNotMatchQuery2 := Query{
-		QueryString: "bla",
-		Filters: []QueryFilter{
-			QueryFilter{FieldName: "someStr", Value: "Pete"},
+		Query{
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someNonexistingField", Value: "Pete", Operator: "!="},
+			},
 		},
 	}
-	shouldNotMatchQuery3 := Query{
-		After: &nextHour,
-	}
-	if !MatchesQuery(lm, shouldMatchQuery) {
-		t.Errorf("Did not match")
-	}
-	if !MatchesQuery(lm, shouldMatchQuery2) {
-		t.Errorf("Did not match 2")
-	}
-	if !MatchesQuery(lm, shouldMatchQuery3) {
-		t.Errorf("Did not match 3")
-	}
-	if !MatchesQuery(lm, shouldMatchQuery4) {
-		t.Errorf("Did not match 4")
-	}
-	if MatchesQuery(lm, shouldNotMatchQuery) {
-		t.Errorf("Did match")
+	shouldNotMatchQueries := []Query{
+		Query{
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="},
+			},
+		},
+		Query{
+			QueryString: "bla",
+			Filters: []QueryFilter{
+				QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="},
+			},
+		},
+		Query{
+			After: &nextHour,
+		},
 	}
-	if MatchesQuery(lm, shouldNotMatchQuery2) {
-		t.Errorf("Did match 2")
+	for i, shouldMatch := range shouldMatchQueries {
+		if !MatchesQuery(lm, shouldMatch) {
+			t.Errorf("Did not match: %d: %+v", i, shouldMatch)
+		}
 	}
-	if MatchesQuery(lm, shouldNotMatchQuery3) {
-		t.Errorf("Did match 3")
+	for i, shouldNotMatch := range shouldNotMatchQueries {
+		if MatchesQuery(lm, shouldNotMatch) {
+			t.Errorf("Did match: %d: %+v", i, shouldNotMatch)
+		}
 	}
 }
 

pkg/backend/kibana/query.go

@@ -42,7 +42,7 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
 	if query.QueryString == "" {
 		queryString = "*"
 	}
-	filterList := JsonList{
+	mustFilters := JsonList{
 		JsonObject{
 			"query_string": JsonObject{
 				"analyze_wildcard": true,
@@ -65,17 +65,29 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
 		if query.Before != nil {
 			rangeObj["range"].(JsonObject)["@timestamp"].(JsonObject)["lt"] = unixMillis(*query.Before)
 		}
-		filterList = append(filterList, rangeObj)
+		mustFilters = append(mustFilters, rangeObj)
 	}
+	mustNotFilters := JsonList{}
 	for _, filter := range query.Filters {
 		m := JsonObject{}
-		m[filter.FieldName] = JsonObject{
-			"query": filter.Value,
-			"type":  "phrase",
+		switch filter.Operator {
+		case "=":
+			m[filter.FieldName] = JsonObject{
+				"query": filter.Value,
+				"type":  "phrase",
+			}
+			mustFilters = append(mustFilters, JsonObject{
+				"match": m,
+			})
+		case "!=":
+			m[filter.FieldName] = JsonObject{
+				"query": filter.Value,
+				"type":  "phrase",
+			}
+			mustNotFilters = append(mustNotFilters, JsonObject{
+				"match": m,
+			})
 		}
-		filterList = append(filterList, JsonObject{
-			"match": m,
-		})
 	}
 	body, err := createMultiSearch(
 		JsonObject{
@@ -94,7 +106,8 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
 			},
 			"query": JsonObject{
 				"bool": JsonObject{
-					"must": filterList,
+					"must":     mustFilters,
+					"must_not": mustNotFilters,
 				},
 			},
 		})
@@ -197,19 +210,20 @@ func (client *Client) querySubIndex(subIndex string, q common.Query) ([]common.L
 
 	allMessages := make([]common.LogMessage, 0, 200)
 	for _, hit := range hits {
-		var attributes map[string]interface{}
-		var ts time.Time
-		attributes = common.Project(hit.Source, q.SelectFields)
-		ts, err = time.Parse(time.RFC3339, hit.Source["@timestamp"].(string))
+		//var ts time.Time
+		attributes := hit.Source
+		ts, err := time.Parse(time.RFC3339, attributes["@timestamp"].(string))
 		if err != nil {
 			return nil, err
 		}
 		delete(attributes, "@timestamp")
-		allMessages = append(allMessages, common.FlattenLogMessage(common.LogMessage{
+		message := common.FlattenLogMessage(common.LogMessage{
 			ID:         hit.ID,
 			Timestamp:  ts,
 			Attributes: attributes,
-		}))
+		})
+		message.Attributes = common.Project(message.Attributes, q.SelectFields)
+		allMessages = append(allMessages, message)
 	}
 	return allMessages, nil
 }