Added ES SimpleClient support for bosun backend and annotation.

[pradeepbbl]

This change will allow to create a light weight elastic search client which is suitable for elasticsearch standalone server.

[pradeepbbl]

Test is failing due to missing annotation backend changes bosun-monitor/annotate#6 are not in place.

[pradeepbbl]

@captncraig after vendor update getting git conflicts

[captncraig]

Oh crap. Yeah, that sucks. Try git checkout --ours vendor/vendor.json and see if that looks ok. (after git merge origin/master)

[pradeepbbl]

@captncraig I tired everything to get get rid of vendor/vendor.json conflict ..am able to merge it in my local repo but it's still failing here. I will retry tom, mean while if you have some free time have look at my version and let me know if you find anything suspicious.

[captncraig]

Here's what I'd recommend.

1. remove all vendor related changes from this branch.
2. rebase onto origin/master
3. make all vendor changes over again.

Sorry, I know this can be real painful.

[pradeepbbl]

@captncraig finally am able to fix the vendor.json conflict issue by replacing the 'revision' hash and 'revisionTime' from origin/master not sure it was the right thing to do?

During the merging process I have noticed go format issue in backend/backend.go, pls have look at bosun-monitor/annotate#7

[kylebrandt]

Here is what I do when working on annotate and vendoring it:

govendor remove github.com/bosun-monitor/annotate/... && govendor add github.com/bosun-monitor/annotate && govendor add github.com/bosun-monitor/annotate/backend && govendor add github.com/bosun-monitor/annotate/web && go run main.go -c bosun.toml -w -r -q -dev -skiplast -n

I think maybe because gorilla stuff things need to be added in a certain
order or something. But the above has always worked for me.

On Tue, Oct 25, 2016 at 5:53 AM, Pradeep Mishra notifications@github.com
wrote:

@captncraig https://github.com/captncraig finally am able to fix the
vendor.json conflict issue by replacing the 'revision' hash and
'revisionTime' from origin/master not sure it was the right thing to do?

[image: screen shot 2016-10-25 at 11 48 57 am]
https://cloud.githubusercontent.com/assets/6874494/19681284/168dd208-9aa9-11e6-9ef6-4e44d73f4390.png

During the merging process I have noticed go format issue in
backend/backend.go, pls have look at bosun-monitor/annotate#7
bosun-monitor/annotate#7

—
You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub
#1947 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/ABnT0McgMZvNUKSM4kKceLCkvHWoP6tEks5q3dGegaJpZM4Ke4MM
.

[pradeepbbl]

this is dependent on bosun-monitor/annotate#7

[pradeepbbl]

force updated vendor/github.com/bosun-monitor/annotate/backend/backend.go by manually replacing the file, which isn't the right way to do but want to make sure all checks are passed.

[pradeepbbl]

@kylebrandt done with code merge for multi elastic backend support :), do let me know if you find any issue or require any further changes.

#1404

Thanks,

[kylebrandt]

@pradeepbbl system tests not passing:

conf/system_test.go:44: cannot use []string literal (type []string) as type map[string]expr.ElasticConfig in field value
conf/system_test.go:44: unknown expr.ElasticHosts field 'SimpleClient' in struct literal

example config used for that test

[kylebrandt]

The following returns "unknown node type" when run on the /expr page:

$minute = 5
$bucketSize = "${minute}m"
$index = esls("logstash")
$keyField = "logsource"
$filter = esquery("message", "scollector")
["default"]escount($index, $keyField, $filter, $bucketSize, "1d", "")

changing the last line to the following works:

avg(["default"]escount($index, $keyField, $filter, $bucketSize, "1d", ""))

[pradeepbbl]

@kylebrandt both above issue has been fixed and pushed 👍 .

sorry forgot to run the test before merging the code :(, do let me know how it goes.

[kylebrandt]

If I mix two sources like the following it works:

$minute = 5
$bucketSize = "${minute}m"
$index = esls("logstash")
$keyField = "logsource"
$ITindex = esls("it-logs")
$ITkeyField = "@source_host"
$filter = esall()
$it = rename(["it"]escount($ITindex, $ITkeyField, $filter, $bucketSize, "3h", ""), "@source_host=logsource")
$default = ["default"]escount($index, $keyField, $filter, $bucketSize, "3h", "")
merge($it, $default)

However, if I don't specify default, I get an error:

$minute = 5
$bucketSize = "${minute}m"
$index = esls("logstash")
$keyField = "logsource"
$ITindex = esls("it-logs")
$ITkeyField = "@source_host"
$filter = esall()
$it = rename(["it"]escount($ITindex, $ITkeyField, $filter, $bucketSize, "3h", ""), "@source_host=logsource")
$default = escount($index, $keyField, $filter, $bucketSize, "3h", "")
merge($it, $default)

[kylebrandt]

// CacheKey returns the text of the elastic query. That text is the indentifer for
// the query in the cache. It is a combination of the indices queries and the json query content
func (r *ElasticRequest) CacheKey() (string, error) {
	s, err := r.Source.Source()
	if err != nil {
		return "", err
	}
	b, err := json.Marshal(s)
	if err != nil {
		return "", fmt.Errorf("failed to generate json representation of search source for cache key: %s", s)
	}

	return fmt.Sprintf("%v\n%s", r.Indices, b), nil
}

// timeESRequest execute the elasticsearch query (which may set or hit cache) and returns
// the search results.
func timeESRequest(e *State, T miniprofiler.Timer, req *ElasticRequest) (resp *elastic.SearchResult, err error) {
	e.elasticQueries = append(e.elasticQueries, *req.Source)
	var source interface{}
	source, err = req.Source.Source()
	if err != nil {
		return resp, fmt.Errorf("failed to get source of request while timing elastic request: %s", err)
	}
	b, err := json.MarshalIndent(source, "", "  ")
	if err != nil {
		return resp, err
	}
	key, err := req.CacheKey()
	if err != nil {
		return nil, err
	}
	T.StepCustomTiming("elastic", "query", fmt.Sprintf("%v\n%s", req.Indices, b), func() {
		getFn := func() (interface{}, error) {
			return e.ElasticHosts.Query(req)
		}
		var val interface{}
		val, err = e.Cache.Get(key, getFn)
		resp = val.(*elastic.SearchResult)
	})
	return
}

Is going to need to identify the server. If this is not the case, identical queries to different servers would be considered the same query you will get only the result of one when it hits the singleflight cache.

[pradeepbbl]

@kylebrandt as suggested changes has been done to add host key to cache. I have made some changes sched/template.go which may require expressions doc change, will do that next week.

[kylebrandt]

Also what about the thing where it doesn't use "default" in that `merge()` example above?

…

On Fri, Feb 24, 2017 at 10:48 AM, Pradeep Mishra ***@***.***> wrote: @kylebrandt <https://github.com/kylebrandt> as suggested changes has been done to add host key to cache. I have made some changes sched/template.go which may require expressions doc change, will do that next week. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#1947 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/ABnT0N4lq4dK3DI72hXTbCBk-3GPxv6Sks5rfvvRgaJpZM4Ke4MM> .

[pradeepbbl]

it working now :)

[pradeepbbl]

if you are also going to test template function (ESQueryAll and ESQuery) with host prefix a new string need to be pass to the function

template test {
	subject = {{.Last.Status}}: {{.Alert.Name}} on {{.Group.host}}
	body = `
	    {{ $filter := (.Eval .Alert.Vars.filter)}}
	    {{ $index := (.Eval .Alert.Vars.index)}}
	    {{ $hostkey := (.Eval .Alert.Vars.prefixkey)}}
	    {{range $i, $x := .ESQuery $index $filter "5m" "" 10 $hostkey}}
	        <p>{{$x.machinename}}</p>
	    {{end}}
	`
}

[kylebrandt]

Need to look at the code, but I think it might be better to assign which backend to use to the Context object attached to each template. That way we don't have to change the arguments to the function. This may not matter so much for the elastic functions, but when we want to add the functionality to more tsdb providers than they won't have to change either.

Since templates are procedural, this should work. You would just do something like `{{ .ES = "mySecondary"}}. Or it could be a method on the context like {{ .UseElastic "mySecondary" }}

[pradeepbbl]

@kylebrandt if we move the variable out from functions to context object expression checks using /expr page are failing with empty prefix key e.g escount($index, $keyField, $filter, $bucketSize, "3h", ""), are you okay to refernece the context obj in expr.go

[kylebrandt]

@pradeepbbl I don't understand why it would need to access the context object in the expr package?

Can't you just call the functions passing the key from the context instead of an argument to the function?

func (c *Context) ESQuery(indexRoot expr.ESIndexer, filter expr.ESQuery, sduration, eduration string, size int, key string) (interface{}, error) {
  	newFilter := expr.ScopeES(c.Group(), filter.Query)		  	newFilter := expr.ScopeES(c.Group(), filter.Query)
 -	req, err := expr.ESBaseQuery(c.runHistory.Start, indexRoot, newFilter, sduration, eduration, size)		 +	req, err := expr.ESBaseQuery(c.runHistory.Start, indexRoot, newFilter, sduration, eduration, size, c.ElasticHost)
...

and then something like:

template test {
	subject = {{.Last.Status}}: {{.Alert.Name}} on {{.Group.host}}
	body = `
	    {{ $filter := (.Eval .Alert.Vars.filter)}}
	    {{ $index := (.Eval .Alert.Vars.index)}}
	    {{ $hostkey := (.Eval .Alert.Vars.prefixkey)}}
            {{ .UseElastic "mySecondary" }}
	    {{range $i, $x := .ESQuery $index $filter "5m" "" 10}}
	        <p>{{$x.machinename}}</p>
	    {{end}}
	`
}

[kylebrandt]

Also, could it be make so the text inside [ ] has to be a quoted string? So [foo] would be invalid but ["foo"] would be valid. Even if we fake it for now and don't actually treat the contents as a string token, at least people won't have differing syntax in their configs should we decide to do so in the future.

[pradeepbbl]

rebase with 'upstream/master'

[pradeepbbl]

@kylebrandt done with the merging and changes discussed above. Please have a look and let me know if anything need to be changed.

Test's Done:

• Merging two different query with and without prefix : pass
• Tested prefix key with double quotes e.g ["foo"]: pass
• Tested prefix key without double quotes e.g [foo]: failed (expected result)

Thanks,

[kylebrandt]

I think we can get rid of the global var here. Initalize it to the string "default" like in https://github.com/bosun-monitor/bosun/pull/1947/files#diff-d6bd3827d8c8d679e462f07cef34f092R755. The for UseElastic have it set c.ElasticHost (and doesn't need to return anything). Since UseElastic is a method on a pointer it will change the context object.

[pradeepbbl]

Sure, I have added it for debug purpose. I will make the change as suggested.

I would also like your suggestion on logging ES host map https://github.com/bosun-monitor/bosun/pull/1947/files#diff-2039e8e0c0f4873ec699c321c1e336f9R261, are you okay with it?

[pradeepbbl]

https://github.com/bookingcom/bosun/blob/85744455d0f56b57bffa21eb89162b5ebb94cd59/cmd/bosun/sched/template.go#L134-L137

[kylebrandt]

this should be removed since it logs whenever the expressions are executed so it will be spammy.

[pradeepbbl]

https://github.com/bookingcom/bosun/blob/85744455d0f56b57bffa21eb89162b5ebb94cd59/cmd/bosun/expr/elastic.go#L259-L261

[kylebrandt]

and remove this one as well.

[kylebrandt]

Oh and documentation. Need to update docs/expression.md to explain the elastic prefix , docs/system_configuration.md with the changes to ES configuration, and definitions.md with the new template func.

[pradeepbbl]

done with the documentation :)

[kylebrandt]

https://github.com/bosun-monitor/bosun/tree/prefixIdea illustrates what I think might be a better way to pass the prefix.

Diff:

diff --git a/cmd/bosun/expr/elastic.go b/cmd/bosun/expr/elastic.go
index 547c74c..aa7948a 100644
--- a/cmd/bosun/expr/elastic.go
+++ b/cmd/bosun/expr/elastic.go
@@ -9,6 +9,7 @@ import (
 	"bosun.org/cmd/bosun/expr/parse"
 	"bosun.org/models"
 	"bosun.org/opentsdb"
+	"bosun.org/slog"
 	"github.com/MiniProfiler/go/miniprofiler"
 	"github.com/jinzhu/now"
 	elastic "gopkg.in/olivere/elastic.v3"
@@ -32,10 +33,11 @@ func elasticTagQuery(args []parse.Node) (parse.Tags, error) {
 var Elastic = map[string]parse.Func{
 	// Funcs for querying elastic
 	"escount": {
-		Args:   []models.FuncType{models.TypeESIndexer, models.TypeString, models.TypeESQuery, models.TypeString, models.TypeString, models.TypeString},
-		Return: models.TypeSeriesSet,
-		Tags:   elasticTagQuery,
-		F:      ESCount,
+		Args:          []models.FuncType{models.TypeESIndexer, models.TypeString, models.TypeESQuery, models.TypeString, models.TypeString, models.TypeString},
+		Return:        models.TypeSeriesSet,
+		Tags:          elasticTagQuery,
+		F:             ESCount,
+		PrefixEnabled: true,
 	},
 	"esstat": {
 		Args:   []models.FuncType{models.TypeESIndexer, models.TypeString, models.TypeESQuery, models.TypeString, models.TypeString, models.TypeString, models.TypeString, models.TypeString},
@@ -419,7 +421,8 @@ func ESMonthly(e *State, T miniprofiler.Timer, timeField, indexRoot, layout stri
 	return &r, nil
 }
 
-func ESCount(e *State, T miniprofiler.Timer, indexer ESIndexer, keystring string, filter ESQuery, interval, sduration, eduration string) (r *Results, err error) {
+func ESCount(prefix string, e *State, T miniprofiler.Timer, indexer ESIndexer, keystring string, filter ESQuery, interval, sduration, eduration string) (r *Results, err error) {
+	slog.Infoln("Prefix: ", prefix)
 	return ESDateHistogram(e, T, indexer, keystring, filter.Query, interval, sduration, eduration, "", "", 0)
 }
 
diff --git a/cmd/bosun/expr/expr.go b/cmd/bosun/expr/expr.go
index 594d6f2..f1d2394 100644
--- a/cmd/bosun/expr/expr.go
+++ b/cmd/bosun/expr/expr.go
@@ -715,9 +715,14 @@ func (e *State) walkPrefix(node *parse.PrefixNode, T miniprofiler.Timer) *Result
 	key, _ = strconv.Unquote(key)
 	switch node := node.Arg.(type) {
 	case *parse.FuncNode:
+		// TODO. Change this to be more generic by having some sort of "supports prefix"
+		// also perhaps better as part of check IIRC (look later)
 		if strings.Contains(node.Name, "es") {
-			e.ElasticHosts.PrefixKey = key
-			prefixkey = true
+			// e.ElasticHosts.PrefixKey = key
+			// prefixkey = true
+
+			// Set the prefix on the func node
+			node.Prefix = key
 		}
 		return e.walk(node, T)
 	default:
@@ -772,7 +777,12 @@ func (e *State) walkFunc(node *parse.FuncNode, T miniprofiler.Timer) *Results {
 		}
 
 		f := reflect.ValueOf(node.F.F)
-		fr := f.Call(append([]reflect.Value{reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
+		fr := []reflect.Value{}
+		if node.F.PrefixEnabled {
+			fr = f.Call(append([]reflect.Value{reflect.ValueOf(node.Prefix), reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
+		} else {
+			fr = f.Call(append([]reflect.Value{reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
+		}
 		res = fr[0].Interface().(*Results)
 		if len(fr) > 1 && !fr[1].IsNil() {
 			err := fr[1].Interface().(error)
diff --git a/cmd/bosun/expr/parse/node.go b/cmd/bosun/expr/parse/node.go
index cc15ee8..1fc6008 100644
--- a/cmd/bosun/expr/parse/node.go
+++ b/cmd/bosun/expr/parse/node.go
@@ -72,6 +72,8 @@ type FuncNode struct {
 	Name string
 	F    Func
 	Args []Node
+	// Prefix is comes from the preceeding prefix node
+	Prefix string
 }
 
 func newFunc(pos Pos, name string, f Func) *FuncNode {
diff --git a/cmd/bosun/expr/parse/parse.go b/cmd/bosun/expr/parse/parse.go
index d71acaf..8f277f9 100644
--- a/cmd/bosun/expr/parse/parse.go
+++ b/cmd/bosun/expr/parse/parse.go
@@ -40,6 +40,7 @@ type Func struct {
 	VArgsPos  int
 	VArgsOmit bool
 	MapFunc   bool // Func is only valid in map expressions
+	PrefixEnabled bool
 	Check     func(*Tree, *FuncNode) error
 }

From Slack private chat:

So if you define a PrefixEnabled: true in map[string]parse.Func then it knows to call F: with the prefix as the first argument

[11:47] 
When parsing the prefix, it passes the value to the funcnode for later reference

[11:49] 
What I didn’t do, is make it so `func (f *FuncNode) Check(t *Tree) error {` checks the FunName against PrefixEnabled

[11:49] 
But that will let you get rid of the if “es”

[kylebrandt]

couple more changes at https://github.com/bosun-monitor/bosun/tree/mesFixes

I need to make sure simple client works with these changes though and test a couple other things. Ironically, one of my ES clusters is down right now.. :-/ Should be fixed in an hour or so

diff --git a/cmd/bosun/expr/elastic.go b/cmd/bosun/expr/elastic.go
index 2d3edab5..ae401b57 100644
--- a/cmd/bosun/expr/elastic.go
+++ b/cmd/bosun/expr/elastic.go
@@ -14,8 +14,12 @@ import (
 	elastic "gopkg.in/olivere/elastic.v3"
 )
 
-// This uses a global client since the elastic client handles connections
-var esClient *elastic.Client
+// Map of prefixes to corresponding clients
+var esClients map[string]*elastic.Client
+
+func init() {
+	esClients = make(map[string]*elastic.Client)
+}
 
 func elasticTagQuery(args []parse.Node) (parse.Tags, error) {
 	n := args[1].(*parse.StringNode)
@@ -252,18 +256,33 @@ type ElasticConfig struct {
 }
 
 // InitClient sets up the elastic client. If the client has already been
-// initalized it is a noop
+// initialized it is a noop
 func (e ElasticHosts) InitClient(prefix string) error {
+	if _, ok := e.Hosts[prefix]; !ok {
+		prefixes := make([]string, len(e.Hosts))
+		i := 0
+		for k := range e.Hosts {
+			prefixes[i] = k
+			i++
+		}
+		return fmt.Errorf("prefix %v not defined, available prefixes are: %v", prefix, prefixes)
+	}
+
+	// TODO? Since SimpleClient isn't "long lived", does that need to be created each time?
+	if c := esClients[prefix]; c != nil {
+		// client already initialized
+		return nil
+	}
 	var err error
 	if e.Hosts[prefix].SimpleClient {
 		// simple client enabled
-		esClient, err = elastic.NewSimpleClient(elastic.SetURL(e.Hosts[prefix].Hosts...), elastic.SetMaxRetries(10))
+		esClients[prefix], err = elastic.NewSimpleClient(elastic.SetURL(e.Hosts[prefix].Hosts...), elastic.SetMaxRetries(10))
 	} else if len(e.Hosts[prefix].Hosts) == 0 {
 		// client option enabled
-		esClient, err = elastic.NewClient(e.Hosts[prefix].ClientOptionFuncs...)
+		esClients[prefix], err = elastic.NewClient(e.Hosts[prefix].ClientOptionFuncs...)
 	} else {
 		// default behavior
-		esClient, err = elastic.NewClient(elastic.SetURL(e.Hosts[prefix].Hosts...), elastic.SetMaxRetries(10))
+		esClients[prefix], err = elastic.NewClient(elastic.SetURL(e.Hosts[prefix].Hosts...), elastic.SetMaxRetries(10))
 	}
 
 	if err != nil {
@@ -278,7 +297,7 @@ func (e *ElasticHosts) getService(prefix string) (*elastic.SearchService, error)
 	if err != nil {
 		return nil, err
 	}
-	return esClient.Search(), nil
+	return esClients[prefix].Search(), nil
 }
 
 // Query takes a Logstash request, applies it a search service, and then queries
@@ -288,9 +307,6 @@ func (e ElasticHosts) Query(r *ElasticRequest) (*elastic.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err != nil {
-		return nil, err
-	}
 
 	s.Index(r.Indices...)
 
diff --git a/cmd/bosun/expr/parse/node.go b/cmd/bosun/expr/parse/node.go
index 0377dd0f..e7b386a8 100644
--- a/cmd/bosun/expr/parse/node.go
+++ b/cmd/bosun/expr/parse/node.go
@@ -317,14 +317,13 @@ func (p *PrefixNode) StringAST() string {
 }
 
 func (p *PrefixNode) Check(t *Tree) error {
-	rt := p.Arg.Return()
-
-	if !(rt == models.TypeSeriesSet) {
-		return fmt.Errorf("parse: type error in %s, expected %s, got %s", p, "prefix", rt)
+	if p.Arg.Type() != NodeFunc {
+		return fmt.Errorf("parse: prefix on non-function")
+	}
+	if !p.Arg.(*FuncNode).F.PrefixEnabled {
+		return fmt.Errorf("func %v does not support a prefix", p.Arg.(*FuncNode).Name)
 	}
-
 	return p.Arg.Check(t)
-
 }
 
 func (p *PrefixNode) Return() models.FuncType {