Added ES client option support along with SimpleClient chnages.

This change will allow to pass different ES client options from bousn configuration file (bosun.toml) as well as support NewSmipleClient (ref: https://github.com/olivere/elastic/blob/release-branch.v3/client.go#L292).

cmd/bosun/bosun.example.toml

@@ -61,12 +61,43 @@ CommandHookPath = "/Users/kbrandt/src/hook/hook"
 # Configuration of hosts to enable the Elastic backend
 [ElasticConf]
 	Hosts = ["http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"]
+	# Set SimpleClient to true if ES running in standalone mode or in a restricted environment
+	# SimpleClient = true
+        # [ElasticConf.ClientOptions]
+            # Enabled = true
+            # BasicAuthUsername = "admin"
+            # BasicAuthPassword = "password"
+
+	        # Default http only support https 
+	        # Scheme = "https" 
+
+            # Default enable 
+            # SnifferEnabled = false
+
+	        # Default enable
+            # HealthcheckEnabled = false
 
 # Configuration for embedding the annotate service (also enables annotations if hosts are defined)
 [AnnotateConf]
     Hosts = ["http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"]
+    # Set SimpleClient to true if ES running in standalone mode or in a restricted environment
+    # SimpleClient = true
     # Set the Index name that annotations are stored in. Default is annotate
     # Index = annotate
+    # [AnnotateConf.ClientOptions]
+            # Enabled = true
+            # BasicAuthUsername = "admin"
+            # BasicAuthPassword = "password"
+
+            # Default http only support https
+            # Scheme = "https"
+
+            # Default enable
+            # SnifferEnabled = false
+
+            # Default enable
+            # HealthcheckEnabled = false
+
 
 # Configuration for Bosun's internal storage. Can be Ledis (Default) or Redis. Redis is recommended
 # for production setups. Defaults for ledis are below but would be ignored since redis takes
@@ -85,4 +116,4 @@ CommandHookPath = "/Users/kbrandt/src/hook/hook"
 [InfluxConf]
 	URL = "https://myInfluxServer:1234"
 	Timeout = "5m"
-	UnsafeSSL = true
+	UnsafeSSL = true

cmd/bosun/conf/system.go

@@ -13,6 +13,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/bosun-monitor/annotate"
 	"github.com/influxdata/influxdb/client"
+	elastic "gopkg.in/olivere/elastic.v3"
 )
 
 // SystemConf contains all the information that bosun needs to run. Outside of the conf package
@@ -96,18 +97,42 @@ type GraphiteConf struct {
 
 // AnnotateConf contains the elastic configuration to enable Annotations support
 type AnnotateConf struct {
-	Hosts []string // CSV of Elastic Hosts, currently the only backend in annotate
-	Index string   // name of index / table
+	Hosts         []string        // CSV of Elastic Hosts, currently the only backend in annotate
+	SimpleClient  bool            // If true ES will connect over NewSimpleClient
+	ClientOptions ESClientOptions // ES client options
+	Index         string          // name of index / table
 }
 
 // LogStashConf contains a list of elastic hosts for the depcrecated logstash functions
 type LogStashConf struct {
 	Hosts expr.LogstashElasticHosts
 }
 
+// ESClientOptions: elastic search client options
+// reference https://github.com/olivere/elastic/blob/release-branch.v3/client.go#L107
+type ESClientOptions struct {
+	Enabled                   bool          // if true use client option else ignore
+	BasicAuthUsername         string        // username for HTTP Basic Auth
+	BasicAuthPassword         string        // password for HTTP Basic Auth
+	Scheme                    string        // https (default http)
+	SnifferEnabled            bool          // sniffer enabled or disabled
+	SnifferTimeoutStartup     time.Duration // in seconds (default is 5 sec)
+	SnifferTimeout            time.Duration // in seconds (default is 2 sec)
+	SnifferInterval           time.Duration // in minutes (default is 15 min)
+	HealthcheckEnabled        bool          // healthchecks enabled or disabled
+	HealthcheckTimeoutStartup time.Duration // in seconds (default is 5 sec)
+	HealthcheckTimeout        time.Duration // in seconds (default is 1 sec)
+	HealthcheckInterval       time.Duration // in seconds (default is 60 sec)
+	MaxRetries                int           // max. number of retries before giving up (default 10)
+	GzipEnabled               bool          // enables or disables gzip compression (disabled by default)
+
+}
+
 // ElasticConf contains configuration for an elastic host that Bosun can query
 type ElasticConf struct {
-	Hosts expr.ElasticHosts
+	Hosts         []string
+	SimpleClient  bool
+	ClientOptions ESClientOptions
 }
 
 // InfluxConf contains configuration for an influx host that Bosun can query
@@ -198,6 +223,12 @@ func loadSystemConfig(conf string, isFileName bool) (*SystemConf, error) {
 	if len(decodeMeta.Undecoded()) > 0 {
 		return sc, fmt.Errorf("undecoded fields in system configuration: %v", decodeMeta.Undecoded())
 	}
+	if sc.ElasticConf.SimpleClient && sc.ElasticConf.ClientOptions.Enabled {
+		return sc, fmt.Errorf("Can't use both ES SimpleClient and ES ClientOptions please remove or disable one in ElasticConf: %#v", sc.ElasticConf)
+	}
+	if sc.AnnotateConf.SimpleClient && sc.AnnotateConf.ClientOptions.Enabled {
+		return sc, fmt.Errorf("Can't use both ES SimpleClient and ES ClientOptions please remove or disable one in AnnotateConf: %#v", sc.AnnotateConf)
+	}
 	sc.md = decodeMeta
 	return sc, nil
 }
@@ -361,7 +392,7 @@ func (sc *SystemConf) GetLogstashElasticHosts() expr.LogstashElasticHosts {
 // GetAnnotateElasticHosts returns the Elastic hosts that should be used for annotations.
 // Annotations are not enabled if this has no hosts
 func (sc *SystemConf) GetAnnotateElasticHosts() expr.ElasticHosts {
-	return sc.AnnotateConf.Hosts
+	return parseESConfig(sc, "annotate")
 }
 
 // GetAnnotateIndex returns the name of the Elastic index that should be used for annotations
@@ -435,7 +466,7 @@ func (sc *SystemConf) GetLogstashContext() expr.LogstashElasticHosts {
 // GetElasticContext returns an Elastic context which contains all the information
 // needed to run Elastic queries.
 func (sc *SystemConf) GetElasticContext() expr.ElasticHosts {
-	return sc.ElasticConf.Hosts
+	return parseESConfig(sc, "elastic")
 }
 
 // AnnotateEnabled returns if annotations have been enabled or not
@@ -479,3 +510,118 @@ func (u *URL) UnmarshalText(text []byte) error {
 	u.URL, err = url.Parse(string(bytes.Trim(text, `\"`)))
 	return err
 }
+
+// ParseESConfig return expr.ElasticHost
+func parseESConfig(sc *SystemConf, config string) expr.ElasticHosts {
+	var options ESClientOptions
+	es_conf := expr.ElasticHosts{}
+	if config == "annotate" {
+		options = sc.AnnotateConf.ClientOptions
+		if !options.Enabled {
+			es_conf.SimpleClient = sc.AnnotateConf.SimpleClient
+			es_conf.Hosts = sc.AnnotateConf.Hosts
+			return es_conf
+		}
+	}
+
+	if config == "elastic" {
+		options = sc.ElasticConf.ClientOptions
+
+		if !options.Enabled {
+			es_conf.SimpleClient = sc.ElasticConf.SimpleClient
+			es_conf.Hosts = sc.ElasticConf.Hosts
+			return es_conf
+		}
+	}
+
+	if options.Enabled {
+		//SetURL
+		es_conf.ClientOptionFuncs = append(
+			es_conf.ClientOptionFuncs,
+			elastic.SetURL(sc.ElasticConf.Hosts...),
+		)
+
+		if options.BasicAuthUsername != "" && options.BasicAuthPassword != "" {
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetBasicAuth(options.BasicAuthUsername, options.BasicAuthPassword),
+			)
+		}
+
+		if options.Scheme == "https" {
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetScheme(options.Scheme),
+			)
+		}
+
+		// Default Enable
+		es_conf.ClientOptionFuncs = append(
+			es_conf.ClientOptionFuncs,
+			elastic.SetSniff(options.SnifferEnabled),
+		)
+
+		if options.SnifferTimeoutStartup > 5 {
+			options.SnifferTimeoutStartup = options.SnifferTimeoutStartup * time.Second
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetSnifferTimeoutStartup(options.SnifferTimeoutStartup),
+			)
+		}
+
+		if options.SnifferTimeout > 2 {
+			options.SnifferTimeout = options.SnifferTimeout * time.Second
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetSnifferTimeout(options.SnifferTimeout),
+			)
+		}
+
+		if options.SnifferInterval > 15 {
+			options.SnifferInterval = options.SnifferInterval * time.Minute
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetSnifferInterval(options.SnifferTimeout),
+			)
+		}
+
+		//Default Enable
+		es_conf.ClientOptionFuncs = append(
+			es_conf.ClientOptionFuncs,
+			elastic.SetHealthcheck(options.HealthcheckEnabled),
+		)
+
+		if options.HealthcheckTimeoutStartup > 5 {
+			options.HealthcheckTimeoutStartup = options.HealthcheckTimeoutStartup * time.Second
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetHealthcheckTimeoutStartup(options.HealthcheckTimeoutStartup),
+			)
+		}
+
+		if options.HealthcheckTimeout > 1 {
+			options.HealthcheckTimeout = options.HealthcheckTimeout * time.Second
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetHealthcheckTimeout(options.HealthcheckTimeout),
+			)
+		}
+
+		if options.HealthcheckInterval > 60 {
+			options.HealthcheckInterval = options.HealthcheckInterval * time.Second
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetHealthcheckInterval(options.HealthcheckInterval),
+			)
+		}
+
+		if options.MaxRetries > 0 {
+			es_conf.ClientOptionFuncs = append(
+				es_conf.ClientOptionFuncs,
+				elastic.SetMaxRetries(options.MaxRetries),
+			)
+		}
+	}
+
+	return es_conf
+}

cmd/bosun/conf/system_test.go

@@ -41,8 +41,9 @@ func TestSystemToml(t *testing.T) {
 		Host:    "localhost:80",
 		Headers: map[string]string{"X-Meow": "Mix"},
 	})
+	c := expr.ElasticHosts{Hosts: []string{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"}, SimpleClient: false}
 	assert.Equal(t, sc.ElasticConf, ElasticConf{
-		Hosts: expr.ElasticHosts{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"},
+		Hosts: c.Hosts,
 	})
 	assert.Equal(t, sc.AnnotateConf, AnnotateConf{
 		Hosts: []string{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"},

cmd/bosun/expr/elastic.go

@@ -238,14 +238,28 @@ func ESLTE(e *State, T miniprofiler.Timer, key string, lte float64) (*Results, e
 // ElasticHosts is an array of Logstash hosts and exists as a type for something to attach
 // methods to.  The elasticsearch library will use the listed to hosts to discover all
 // of the hosts in the config
-type ElasticHosts []string
+// type ElasticHosts []string
+type ElasticHosts struct {
+	Hosts             []string
+	SimpleClient      bool
+	ClientOptionFuncs []elastic.ClientOptionFunc
+}
 
 // InitClient sets up the elastic client. If the client has already been
 // initalized it is a noop
 func (e ElasticHosts) InitClient() error {
 	if esClient == nil {
 		var err error
-		esClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10))
+		if e.SimpleClient {
+			// simple client enabled
+			esClient, err = elastic.NewSimpleClient(elastic.SetURL(e.Hosts...), elastic.SetMaxRetries(10))
+		} else if len(e.Hosts) == 0 {
+			// client option enabled
+			esClient, err = elastic.NewClient(e.ClientOptionFuncs...)
+		} else {
+			// default behavior
+			esClient, err = elastic.NewClient(elastic.SetURL(e.Hosts...), elastic.SetMaxRetries(10))
+		}
 		if err != nil {
 			return err
 		}

cmd/bosun/web/web.go

@@ -147,7 +147,8 @@ func Listen(listenAddr string, devMode bool, tsdbHost string, reloadFunc func()
 		if index == "" {
 			index = "annotate"
 		}
-		annotateBackend = backend.NewElastic(schedule.SystemConf.GetAnnotateElasticHosts(), index)
+		c := schedule.SystemConf.GetAnnotateElasticHosts()
+		annotateBackend = backend.NewElastic(c.Hosts, c.SimpleClient, c.ClientOptionFuncs, index)
 
 		go func() {
 			for {

vendor/vendor.json

@@ -254,16 +254,19 @@
 			"revisionTime": "2015-11-22T20:06:43-07:00"
 		},
 		{
+			"checksumSHA1": "jx1hiVjE68LP+EMqgSiOJ5udPCQ=",
 			"path": "github.com/bosun-monitor/annotate",
 			"revision": "7a63972472cdcbd52933b5580927efca995e4598",
 			"revisionTime": "2016-10-19T16:13:39-06:00"
 		},
 		{
+			"checksumSHA1": "/N7Fn2S5O4gcqloH5fdkGzJ3Gas=",
 			"path": "github.com/bosun-monitor/annotate/backend",
 			"revision": "7a63972472cdcbd52933b5580927efca995e4598",
 			"revisionTime": "2016-10-19T16:13:39-06:00"
 		},
 		{
+			"checksumSHA1": "GdXtpLoHxfTdgsdgr3XaEmT7eCk=",
 			"path": "github.com/bosun-monitor/annotate/web",
 			"revision": "7a63972472cdcbd52933b5580927efca995e4598",
 			"revisionTime": "2016-10-19T16:13:39-06:00"