Ax

Makefile

@@ -0,0 +1,8 @@
+all:
+	go install github.com/zefhemel/ax/cmd/ax
+
+test:
+	go test ./cmd/... ./pkg/...
+
+deps:
+	go get ./cmd/... ./pkg/...

cmd/ax/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+	"strings"
+
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/zefhemel/ax/pkg/backend/common"
+	"github.com/zefhemel/ax/pkg/backend/docker"
+	"github.com/zefhemel/ax/pkg/backend/kibana"
+	"github.com/zefhemel/ax/pkg/backend/stream"
+	"github.com/zefhemel/ax/pkg/backend/subprocess"
+	"github.com/zefhemel/ax/pkg/config"
+)
+
+var (
+	queryCommand = kingpin.Command("query", "Query logs").Default()
+)
+
+func main() {
+	stat, _ := os.Stdin.Stat()
+	cmd := kingpin.Parse()
+
+	rc := config.BuildConfig()
+	var client common.Client
+	if (stat.Mode() & os.ModeCharDevice) == 0 {
+		client = stream.New(os.Stdin)
+	} else if rc.Env["backend"] == "docker" {
+		client = docker.New(rc.Env["pattern"])
+	} else if rc.Env["backend"] == "kibana" {
+		client = kibana.New(rc.Env["url"], rc.Env["auth"], rc.Env["index"])
+	} else if rc.Env["backend"] == "subprocess" {
+		client = subprocess.New(strings.Split(rc.Env["command"], " "))
+	}
+
+	switch cmd {
+	case "query":
+		queryMain(rc, client)
+	case "init":
+		config.InitSetup()
+	}
+
+}

cmd/ax/query.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/araddon/dateparse"
+	"github.com/fatih/color"
+	"github.com/zefhemel/ax/pkg/backend/common"
+	"github.com/zefhemel/ax/pkg/complete"
+	"github.com/zefhemel/ax/pkg/config"
+	yaml "gopkg.in/yaml.v2"
+)
+
+var (
+	queryBefore     = queryCommand.Flag("before", "Results from before").String()
+	queryAfter      = queryCommand.Flag("after", "Results from after").String()
+	queryMaxResults = queryCommand.Flag("results", "Maximum number of results").Short('n').Default("200").Int()
+	querySelect     = queryCommand.Flag("select", "Fields to select").Short('s').HintAction(selectHintAction).Strings()
+	queryWhere      = queryCommand.Flag("where", "Add a filter").Short('w').HintAction(whereHintAction).Strings()
+	//querySortDesc     = queryCommand.Flag("desc", "Sort results reverse-chronologically").Default("false").Bool()
+	queryOutputFormat = queryCommand.Flag("output", "Output format: text|json|yaml").Short('o').Default("text").Enum("text", "yaml", "json", "pretty-json")
+	queryFollow       = queryCommand.Flag("follow", "Follow log in quasi-realtime, similar to tail -f").Short('f').Default("false").Bool()
+	queryString       = queryCommand.Arg("query", "Query string").Default("").Strings()
+)
+
+func whereHintAction() []string {
+	rc := config.BuildConfig()
+	resultList := make([]string, 0, 20)
+	for attrName, _ := range complete.GetCompletions(rc) {
+		resultList = append(resultList, fmt.Sprintf("%s=", attrName))
+	}
+	return resultList
+}
+
+func selectHintAction() []string {
+	rc := config.BuildConfig()
+	resultList := make([]string, 0, 20)
+	for attrName, _ := range complete.GetCompletions(rc) {
+		resultList = append(resultList, attrName)
+	}
+	return resultList
+}
+
+func buildFilters(wheres []string) []common.QueryFilter {
+	filters := make([]common.QueryFilter, 0, len(wheres))
+	for _, whereClause := range wheres {
+		pieces := strings.SplitN(whereClause, "=", 2)
+		if len(pieces) != 2 {
+			fmt.Println("Invalid where clause", whereClause)
+			os.Exit(1)
+		}
+		filters = append(filters, common.QueryFilter{
+			FieldName: pieces[0],
+			Value:     pieces[1],
+		})
+	}
+	return filters
+}
+
+func queryMain(rc config.RuntimeConfig, client common.Client) {
+	var before *time.Time
+	var after *time.Time
+	if *queryAfter != "" {
+		var err error
+		afterTime, err := dateparse.ParseLocal(*queryAfter)
+		if err != nil {
+			fmt.Println("Could parse after date:", *queryAfter)
+			os.Exit(1)
+		}
+		fmt.Fprintf(os.Stderr, "Parsed --after as %s", afterTime.Format(time.RFC3339))
+		after = &afterTime
+	}
+	if *queryBefore != "" {
+		var err error
+		beforeTime, err := dateparse.ParseLocal(*queryBefore)
+		if err != nil {
+			fmt.Println("Could parse before date:", *queryBefore)
+			os.Exit(1)
+		}
+		fmt.Fprintf(os.Stderr, "Parsed --before as %s", beforeTime.Format(time.RFC3339))
+		before = &beforeTime
+	}
+
+	for message := range complete.GatherCompletionInfo(rc, client.Query(common.Query{
+		QueryString:  strings.Join(*queryString, " "),
+		Before:       before,
+		After:        after,
+		Filters:      buildFilters(*queryWhere),
+		MaxResults:   *queryMaxResults,
+		SelectFields: *querySelect,
+		Follow:       *queryFollow,
+	})) {
+		printMessage(message, *queryOutputFormat)
+	}
+
+}
+
+func printMessage(message common.LogMessage, queryOutputFormat string) {
+	switch queryOutputFormat {
+	case "text":
+		ts := message.Timestamp.Format(time.RFC3339)
+		fmt.Printf("[%s] ", color.MagentaString(ts))
+		if message.Message != "" {
+			messageColor := color.New(color.Bold)
+			fmt.Printf("%s ", messageColor.Sprint(message.Message))
+		}
+		for key, value := range message.Attributes {
+			fmt.Printf("%s=%s ", color.CyanString(key), common.MustJsonEncode(value))
+		}
+		fmt.Println()
+	case "json":
+		encoder := json.NewEncoder(os.Stdout)
+		err := encoder.Encode(message)
+		if err != nil {
+			fmt.Println("Error JSON encoding")
+		}
+	case "json-pretty":
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		err := encoder.Encode(message)
+		if err != nil {
+			fmt.Println("Error JSON encoding")
+		}
+	case "yaml":
+		buf, err := yaml.Marshal(message)
+		if err != nil {
+			fmt.Println("Error YAML encoding")
+		}
+		fmt.Printf("---\n%s", string(buf))
+	}
+}

pkg/backend/common/client.go

@@ -0,0 +1,127 @@
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+type Client interface {
+	Query(query Query) <-chan LogMessage
+}
+
+type QueryFilter struct {
+	FieldName string
+	Value     string
+}
+
+type Query struct {
+	QueryString  string
+	After        *time.Time
+	Before       *time.Time
+	SelectFields []string
+	Filters      []QueryFilter
+	MaxResults   int
+	// QueryAsc     bool
+	// ResultsDesy  bool
+	Follow bool
+}
+
+type LogMessage struct {
+	Timestamp  time.Time
+	Message    string
+	Attributes map[string]interface{}
+}
+
+func NewLogMessage() LogMessage {
+	return LogMessage{
+		Attributes: make(map[string]interface{}),
+	}
+}
+
+func MustJsonEncode(obj interface{}) string {
+	buf, err := json.Marshal(obj)
+	if err != nil {
+		panic(err)
+	}
+	return string(buf)
+}
+
+func MustJsonDecode(jsonString string, dst interface{}) {
+	decoder := json.NewDecoder(strings.NewReader(jsonString))
+	err := decoder.Decode(dst)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func FlattenAttributes(m, into map[string]interface{}, prefix string) {
+	for k, gv := range m {
+		switch v := gv.(type) {
+		case map[string]interface{}:
+			FlattenAttributes(v, into, fmt.Sprintf("%s%s.", prefix, k))
+		default:
+			into[fmt.Sprintf("%s%s", prefix, k)] = gv
+		}
+	}
+}
+
+func FlattenLogMessage(message LogMessage) LogMessage {
+	newMessage := NewLogMessage()
+	newMessage.Timestamp = message.Timestamp
+	newMessage.Message = message.Message
+	FlattenAttributes(message.Attributes, newMessage.Attributes, "")
+	return newMessage
+}
+
+func Project(m map[string]interface{}, fields []string) map[string]interface{} {
+	if len(fields) == 0 {
+		return m
+	}
+	projected := make(map[string]interface{})
+	for _, field := range fields {
+		if val, ok := m[field]; ok {
+			projected[field] = val
+		}
+	}
+	return projected
+}
+
+func (f QueryFilter) Matches(m LogMessage) bool {
+	val, ok := m.Attributes[f.FieldName]
+	return ok && f.Value == fmt.Sprintf("%v", val)
+}
+
+func matchesPhrase(s, phrase string) bool {
+	return strings.Contains(strings.ToLower(s), strings.ToLower(phrase))
+}
+
+func MatchesQuery(m LogMessage, q Query) bool {
+	matchFound := matchesPhrase(m.Message, q.QueryString)
+	if q.QueryString != "" {
+		for _, v := range m.Attributes {
+			if vs, ok := v.(string); ok {
+				if matchesPhrase(vs, q.QueryString) {
+					matchFound = true
+				}
+			}
+		}
+	}
+	if q.Before != nil {
+		if m.Timestamp.After(*q.Before) {
+			return false
+		}
+	}
+	if q.After != nil {
+		if m.Timestamp.Before(*q.After) {
+			return false
+		}
+	}
+	for _, f := range q.Filters {
+		if !f.Matches(m) {
+			return false
+		}
+	}
+	return matchFound
+}