From c49681e21f5d4a278ce8220bb856df61a33945d7 Mon Sep 17 00:00:00 2001 From: Pradeep Mishra Date: Mon, 24 Oct 2016 16:37:21 +0200 Subject: [PATCH] Added ES client option support along with SimpleClient chnages. This change will allow to pass different ES client options from bousn configuration file (bosun.toml) as well as support NewSmipleClient (ref: https://github.com/olivere/elastic/blob/release-branch.v3/client.go#L292). --- cmd/bosun/bosun.example.toml | 33 +++- cmd/bosun/conf/system.go | 156 +++++++++++++++++- cmd/bosun/conf/system_test.go | 3 +- cmd/bosun/expr/elastic.go | 18 +- cmd/bosun/web/web.go | 3 +- .../bosun-monitor/annotate/backend/backend.go | 23 ++- vendor/vendor.json | 3 + 7 files changed, 222 insertions(+), 17 deletions(-) diff --git a/cmd/bosun/bosun.example.toml b/cmd/bosun/bosun.example.toml index 3710463243..9688e25409 100644 --- a/cmd/bosun/bosun.example.toml +++ b/cmd/bosun/bosun.example.toml @@ -61,12 +61,43 @@ CommandHookPath = "/Users/kbrandt/src/hook/hook" # Configuration of hosts to enable the Elastic backend [ElasticConf] Hosts = ["http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"] + # Set SimpleClient to true if ES running in standalone mode or in a restricted environment + # SimpleClient = true + # [ElasticConf.ClientOptions] + # Enabled = true + # BasicAuthUsername = "admin" + # BasicAuthPassword = "password" + + # Default http only support https + # Scheme = "https" + + # Default enable + # SnifferEnabled = false + + # Default enable + # HealthcheckEnabled = false # Configuration for embedding the annotate service (also enables annotations if hosts are defined) [AnnotateConf] Hosts = ["http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"] + # Set SimpleClient to true if ES running in standalone mode or in a restricted environment + # SimpleClient = true # Set the Index name that annotations are stored in. Default is annotate # Index = annotate + # [AnnotateConf.ClientOptions] + # Enabled = true + # BasicAuthUsername = "admin" + # BasicAuthPassword = "password" + + # Default http only support https + # Scheme = "https" + + # Default enable + # SnifferEnabled = false + + # Default enable + # HealthcheckEnabled = false + # Configuration for Bosun's internal storage. Can be Ledis (Default) or Redis. Redis is recommended # for production setups. Defaults for ledis are below but would be ignored since redis takes @@ -85,4 +116,4 @@ CommandHookPath = "/Users/kbrandt/src/hook/hook" [InfluxConf] URL = "https://myInfluxServer:1234" Timeout = "5m" - UnsafeSSL = true \ No newline at end of file + UnsafeSSL = true diff --git a/cmd/bosun/conf/system.go b/cmd/bosun/conf/system.go index da5b7be107..8d521f0ac9 100644 --- a/cmd/bosun/conf/system.go +++ b/cmd/bosun/conf/system.go @@ -13,6 +13,7 @@ import ( "github.com/BurntSushi/toml" "github.com/bosun-monitor/annotate" "github.com/influxdata/influxdb/client" + elastic "gopkg.in/olivere/elastic.v3" ) // SystemConf contains all the information that bosun needs to run. Outside of the conf package @@ -96,8 +97,10 @@ type GraphiteConf struct { // AnnotateConf contains the elastic configuration to enable Annotations support type AnnotateConf struct { - Hosts []string // CSV of Elastic Hosts, currently the only backend in annotate - Index string // name of index / table + Hosts []string // CSV of Elastic Hosts, currently the only backend in annotate + SimpleClient bool // If true ES will connect over NewSimpleClient + ClientOptions ESClientOptions // ES client options + Index string // name of index / table } // LogStashConf contains a list of elastic hosts for the depcrecated logstash functions @@ -105,9 +108,31 @@ type LogStashConf struct { Hosts expr.LogstashElasticHosts } +// ESClientOptions: elastic search client options +// reference https://github.com/olivere/elastic/blob/release-branch.v3/client.go#L107 +type ESClientOptions struct { + Enabled bool // if true use client option else ignore + BasicAuthUsername string // username for HTTP Basic Auth + BasicAuthPassword string // password for HTTP Basic Auth + Scheme string // https (default http) + SnifferEnabled bool // sniffer enabled or disabled + SnifferTimeoutStartup time.Duration // in seconds (default is 5 sec) + SnifferTimeout time.Duration // in seconds (default is 2 sec) + SnifferInterval time.Duration // in minutes (default is 15 min) + HealthcheckEnabled bool // healthchecks enabled or disabled + HealthcheckTimeoutStartup time.Duration // in seconds (default is 5 sec) + HealthcheckTimeout time.Duration // in seconds (default is 1 sec) + HealthcheckInterval time.Duration // in seconds (default is 60 sec) + MaxRetries int // max. number of retries before giving up (default 10) + GzipEnabled bool // enables or disables gzip compression (disabled by default) + +} + // ElasticConf contains configuration for an elastic host that Bosun can query type ElasticConf struct { - Hosts expr.ElasticHosts + Hosts []string + SimpleClient bool + ClientOptions ESClientOptions } // InfluxConf contains configuration for an influx host that Bosun can query @@ -198,6 +223,12 @@ func loadSystemConfig(conf string, isFileName bool) (*SystemConf, error) { if len(decodeMeta.Undecoded()) > 0 { return sc, fmt.Errorf("undecoded fields in system configuration: %v", decodeMeta.Undecoded()) } + if sc.ElasticConf.SimpleClient && sc.ElasticConf.ClientOptions.Enabled { + return sc, fmt.Errorf("Can't use both ES SimpleClient and ES ClientOptions please remove or disable one in ElasticConf: %#v", sc.ElasticConf) + } + if sc.AnnotateConf.SimpleClient && sc.AnnotateConf.ClientOptions.Enabled { + return sc, fmt.Errorf("Can't use both ES SimpleClient and ES ClientOptions please remove or disable one in AnnotateConf: %#v", sc.AnnotateConf) + } sc.md = decodeMeta return sc, nil } @@ -361,7 +392,7 @@ func (sc *SystemConf) GetLogstashElasticHosts() expr.LogstashElasticHosts { // GetAnnotateElasticHosts returns the Elastic hosts that should be used for annotations. // Annotations are not enabled if this has no hosts func (sc *SystemConf) GetAnnotateElasticHosts() expr.ElasticHosts { - return sc.AnnotateConf.Hosts + return parseESConfig(sc, "annotate") } // GetAnnotateIndex returns the name of the Elastic index that should be used for annotations @@ -435,7 +466,7 @@ func (sc *SystemConf) GetLogstashContext() expr.LogstashElasticHosts { // GetElasticContext returns an Elastic context which contains all the information // needed to run Elastic queries. func (sc *SystemConf) GetElasticContext() expr.ElasticHosts { - return sc.ElasticConf.Hosts + return parseESConfig(sc, "elastic") } // AnnotateEnabled returns if annotations have been enabled or not @@ -479,3 +510,118 @@ func (u *URL) UnmarshalText(text []byte) error { u.URL, err = url.Parse(string(bytes.Trim(text, `\"`))) return err } + +// ParseESConfig return expr.ElasticHost +func parseESConfig(sc *SystemConf, config string) expr.ElasticHosts { + var options ESClientOptions + es_conf := expr.ElasticHosts{} + if config == "annotate" { + options = sc.AnnotateConf.ClientOptions + if !options.Enabled { + es_conf.SimpleClient = sc.AnnotateConf.SimpleClient + es_conf.Hosts = sc.AnnotateConf.Hosts + return es_conf + } + } + + if config == "elastic" { + options = sc.ElasticConf.ClientOptions + + if !options.Enabled { + es_conf.SimpleClient = sc.ElasticConf.SimpleClient + es_conf.Hosts = sc.ElasticConf.Hosts + return es_conf + } + } + + if options.Enabled { + //SetURL + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetURL(sc.ElasticConf.Hosts...), + ) + + if options.BasicAuthUsername != "" && options.BasicAuthPassword != "" { + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetBasicAuth(options.BasicAuthUsername, options.BasicAuthPassword), + ) + } + + if options.Scheme == "https" { + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetScheme(options.Scheme), + ) + } + + // Default Enable + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetSniff(options.SnifferEnabled), + ) + + if options.SnifferTimeoutStartup > 5 { + options.SnifferTimeoutStartup = options.SnifferTimeoutStartup * time.Second + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetSnifferTimeoutStartup(options.SnifferTimeoutStartup), + ) + } + + if options.SnifferTimeout > 2 { + options.SnifferTimeout = options.SnifferTimeout * time.Second + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetSnifferTimeout(options.SnifferTimeout), + ) + } + + if options.SnifferInterval > 15 { + options.SnifferInterval = options.SnifferInterval * time.Minute + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetSnifferInterval(options.SnifferTimeout), + ) + } + + //Default Enable + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetHealthcheck(options.HealthcheckEnabled), + ) + + if options.HealthcheckTimeoutStartup > 5 { + options.HealthcheckTimeoutStartup = options.HealthcheckTimeoutStartup * time.Second + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetHealthcheckTimeoutStartup(options.HealthcheckTimeoutStartup), + ) + } + + if options.HealthcheckTimeout > 1 { + options.HealthcheckTimeout = options.HealthcheckTimeout * time.Second + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetHealthcheckTimeout(options.HealthcheckTimeout), + ) + } + + if options.HealthcheckInterval > 60 { + options.HealthcheckInterval = options.HealthcheckInterval * time.Second + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetHealthcheckInterval(options.HealthcheckInterval), + ) + } + + if options.MaxRetries > 0 { + es_conf.ClientOptionFuncs = append( + es_conf.ClientOptionFuncs, + elastic.SetMaxRetries(options.MaxRetries), + ) + } + } + + return es_conf +} diff --git a/cmd/bosun/conf/system_test.go b/cmd/bosun/conf/system_test.go index 459e67b68f..bc8ad549ea 100644 --- a/cmd/bosun/conf/system_test.go +++ b/cmd/bosun/conf/system_test.go @@ -41,8 +41,9 @@ func TestSystemToml(t *testing.T) { Host: "localhost:80", Headers: map[string]string{"X-Meow": "Mix"}, }) + c := expr.ElasticHosts{Hosts: []string{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"}, SimpleClient: false} assert.Equal(t, sc.ElasticConf, ElasticConf{ - Hosts: expr.ElasticHosts{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"}, + Hosts: c.Hosts, }) assert.Equal(t, sc.AnnotateConf, AnnotateConf{ Hosts: []string{"http://ny-lselastic01.example.com:9200", "http://ny-lselastic02.example.com:9200"}, diff --git a/cmd/bosun/expr/elastic.go b/cmd/bosun/expr/elastic.go index cf91cb806a..41ab4766e1 100644 --- a/cmd/bosun/expr/elastic.go +++ b/cmd/bosun/expr/elastic.go @@ -238,14 +238,28 @@ func ESLTE(e *State, T miniprofiler.Timer, key string, lte float64) (*Results, e // ElasticHosts is an array of Logstash hosts and exists as a type for something to attach // methods to. The elasticsearch library will use the listed to hosts to discover all // of the hosts in the config -type ElasticHosts []string +// type ElasticHosts []string +type ElasticHosts struct { + Hosts []string + SimpleClient bool + ClientOptionFuncs []elastic.ClientOptionFunc +} // InitClient sets up the elastic client. If the client has already been // initalized it is a noop func (e ElasticHosts) InitClient() error { if esClient == nil { var err error - esClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10)) + if e.SimpleClient { + // simple client enabled + esClient, err = elastic.NewSimpleClient(elastic.SetURL(e.Hosts...), elastic.SetMaxRetries(10)) + } else if len(e.Hosts) == 0 { + // client option enabled + esClient, err = elastic.NewClient(e.ClientOptionFuncs...) + } else { + // default behavior + esClient, err = elastic.NewClient(elastic.SetURL(e.Hosts...), elastic.SetMaxRetries(10)) + } if err != nil { return err } diff --git a/cmd/bosun/web/web.go b/cmd/bosun/web/web.go index 3672b96db5..331165d4fe 100644 --- a/cmd/bosun/web/web.go +++ b/cmd/bosun/web/web.go @@ -147,7 +147,8 @@ func Listen(listenAddr string, devMode bool, tsdbHost string, reloadFunc func() if index == "" { index = "annotate" } - annotateBackend = backend.NewElastic(schedule.SystemConf.GetAnnotateElasticHosts(), index) + c := schedule.SystemConf.GetAnnotateElasticHosts() + annotateBackend = backend.NewElastic(c.Hosts, c.SimpleClient, c.ClientOptionFuncs, index) go func() { for { diff --git a/vendor/github.com/bosun-monitor/annotate/backend/backend.go b/vendor/github.com/bosun-monitor/annotate/backend/backend.go index 85f0a383c1..64bec26eb0 100644 --- a/vendor/github.com/bosun-monitor/annotate/backend/backend.go +++ b/vendor/github.com/bosun-monitor/annotate/backend/backend.go @@ -25,14 +25,15 @@ const docType = "annotation" type Elastic struct { *elastic.Client - index string - urls []string - maxResults int - initialized bool + index string + urls []string + simpleClient bool + maxResults int + initialized bool } -func NewElastic(urls []string, index string) *Elastic { - return &Elastic{&elastic.Client{}, index, urls, 200, false} +func NewElastic(urls []string, simpleclient bool, index string) *Elastic { + return &Elastic{&elastic.Client{}, index, urls, simpleclient, 200, false} } var unInitErr = fmt.Errorf("backend has not been initialized") @@ -172,7 +173,15 @@ func (e *Elastic) GetFieldValues(field string) ([]string, error) { } func (e *Elastic) InitBackend() error { - ec, err := elastic.NewClient(elastic.SetURL(e.urls...)) + var err error + var ec *elastic.Client + + if e.simpleClient { + ec, err = elastic.NewSimpleClient(elastic.SetURL(e.urls...)) + } else { + ec, err = elastic.NewClient(elastic.SetURL(e.urls...)) + } + if err != nil { return err } diff --git a/vendor/vendor.json b/vendor/vendor.json index df95100349..29b34fd26f 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -254,16 +254,19 @@ "revisionTime": "2015-11-22T20:06:43-07:00" }, { + "checksumSHA1": "jx1hiVjE68LP+EMqgSiOJ5udPCQ=", "path": "github.com/bosun-monitor/annotate", "revision": "7a63972472cdcbd52933b5580927efca995e4598", "revisionTime": "2016-10-19T16:13:39-06:00" }, { + "checksumSHA1": "/N7Fn2S5O4gcqloH5fdkGzJ3Gas=", "path": "github.com/bosun-monitor/annotate/backend", "revision": "7a63972472cdcbd52933b5580927efca995e4598", "revisionTime": "2016-10-19T16:13:39-06:00" }, { + "checksumSHA1": "GdXtpLoHxfTdgsdgr3XaEmT7eCk=", "path": "github.com/bosun-monitor/annotate/web", "revision": "7a63972472cdcbd52933b5580927efca995e4598", "revisionTime": "2016-10-19T16:13:39-06:00"