Master

.github/workflows/golangci-lint.yml

@@ -0,0 +1,31 @@
+name: golangci-lint
+on:
+  push:
+    paths:
+      - "go.sum"
+      - "go.mod"
+      - "**.go"
+      - "scripts/errcheck_excludes.txt"
+      - ".github/workflows/golangci-lint.yml"
+      - ".golangci.yml"
+  pull_request:
+    paths:
+      - "go.sum"
+      - "go.mod"
+      - "**.go"
+      - "scripts/errcheck_excludes.txt"
+      - ".github/workflows/golangci-lint.yml"
+      - ".golangci.yml"
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.42.0

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.3.0 / 2021-10-21
+
+* [FEATURE] Add support for passing elasticsearch credentials via the ES_USERNAME and ES_PASSWORD environment varialbes #461
+* [FEATURE] Add support for API keys for elasticsearch authentication (Elastic cloud) #459
+* [BUGFIX] Fix index stats when shards are unavailable #445
+
 ## 1.2.1 / 2021-06-29
 
 * [BUGFIX] Fixed elasticsearch 7.13 node stats metrics #439

Makefile.common

@@ -83,12 +83,18 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.39.0
+GOLANGCI_LINT_VERSION ?= v1.42.0
 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
 	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
-		GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		# If we're in CI and there is an Actions file, that means the linter
+		# is being run in Actions, so we don't need to run it here.
+		ifeq (,$(CIRCLE_JOB))
+			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		else ifeq (,$(wildcard .github/workflows/golangci-lint.yml))
+			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		endif
 	endif
 endif
 
@@ -154,7 +160,7 @@ endif
 update-go-deps:
 	@echo ">> updating Go dependencies"
 	@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
-		$(GO) get $$m; \
+		$(GO) get -d $$m; \
 	done
 	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
 ifneq (,$(wildcard vendor))

README.md

@@ -48,6 +48,7 @@ elasticsearch_exporter --help
 | es.cluster_settings     | 1.1.0rc1              | If true, query stats for cluster settings. | false |
 | es.indices              | 1.0.2                 | If true, query stats for all indices in the cluster. | false |
 | es.indices_settings     | 1.0.4rc1              | If true, query settings stats for all indices in the cluster. | false |
+| es.indices_mappings     | 1.2.0                 | If true, query stats for mappings of all indices of the cluster. | false |
 | es.shards               | 1.0.3rc1              | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false |
 | es.snapshots            | 1.0.4rc1              | If true, query stats for the cluster snapshots. | false |
 | es.timeout              | 1.0.2                 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s |
@@ -56,6 +57,7 @@ elasticsearch_exporter --help
 | es.client-cert          | 1.0.2                 | Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch. | |
 | es.clusterinfo.interval | 1.1.0rc1              |  Cluster info update interval for the cluster label | 5m |
 | es.ssl-skip-verify      | 1.0.4rc1              | Skip SSL verification when connecting to Elasticsearch. | false |
+| es.apiKey               | unreleased            | API Key to use for authenticating against Elasticsearch. |  |
 | web.listen-address      | 1.0.2                 | Address to listen on for web interface and telemetry. | :9114 |
 | web.telemetry-path      | 1.0.2                 | Path under which to expose metrics. | /metrics |
 | version                 | 1.0.2                 | Show version info on stdout and exit. | |
@@ -66,6 +68,9 @@ by replacing `.` and `-` with `_` and upper-casing the parameter name.
 
 #### Elasticsearch 7.x security privileges
 
+Username and password can be passed either directly in the URI or through the `ES_USERNAME` and `ES_PASSWORD` environment variables.
+Specifying those two environment variables will override authentication passed in the URI (if any).
+
 ES 7.x supports RBACs. The following security privileges are required for the elasticsearch_exporter.
 
 Setting | Privilege Required | Description
@@ -111,6 +116,7 @@ Further Information
 | elasticsearch_filesystem_io_stats_device_write_operations_count       | gauge     | 1           | Count of disk write operations
 | elasticsearch_filesystem_io_stats_device_read_size_kilobytes_sum      | gauge     | 1           | Total kilobytes read from disk
 | elasticsearch_filesystem_io_stats_device_write_size_kilobytes_sum     | gauge     | 1           | Total kilobytes written to disk
+| elasticsearch_indices_active_queries                                  | gauge     | 1           | The number of currently active queries
 | elasticsearch_indices_docs                                            | gauge     | 1           | Count of documents on this node
 | elasticsearch_indices_docs_deleted                                    | gauge     | 1           | Count of deleted documents on this node
 | elasticsearch_indices_docs_primary                                    | gauge     |             | Count of documents with only primary shards on all nodes
@@ -128,6 +134,7 @@ Further Information
 | elasticsearch_indices_get_total                                       | counter   | 1           | Total get
 | elasticsearch_indices_indexing_delete_time_seconds_total              | counter   | 1           | Total time indexing delete in seconds
 | elasticsearch_indices_indexing_delete_total                           | counter   | 1           | Total indexing deletes
+| elasticsearch_indices_index_current                                   | gauge     | 1           | The number of documents currently being indexed to an index
 | elasticsearch_indices_indexing_index_time_seconds_total               | counter   | 1           | Cumulative index time in seconds
 | elasticsearch_indices_indexing_index_total                            | counter   | 1           | Total index calls
 | elasticsearch_indices_mappings_stats_fields                           | gauge     | 1           | Count of fields currently mapped by index

VERSION

@@ -1 +1 @@
-1.2.1
+1.3.0

collector/cluster_health.go

@@ -21,8 +21,8 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

collector/cluster_health_test.go

@@ -20,7 +20,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestClusterHealth(t *testing.T) {

collector/cluster_settings.go

@@ -22,8 +22,8 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/imdario/mergo"
 	"github.com/prometheus/client_golang/prometheus"
 )

collector/cluster_settings_test.go

@@ -21,7 +21,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestClusterSettingsStats(t *testing.T) {

collector/indices.go

@@ -22,8 +22,8 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -491,6 +491,18 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo
 				},
 				Labels: indexLabels,
 			},
+			{
+				Type: prometheus.GaugeValue,
+				Desc: prometheus.NewDesc(
+					prometheus.BuildFQName(namespace, "search", "active_queries"),
+					"The number of currently active queries",
+					indexLabels.keys(), nil,
+				),
+				Value: func(indexStats IndexStatsIndexResponse) float64 {
+					return float64(indexStats.Total.Search.QueryCurrent)
+				},
+				Labels: indexLabels,
+			},
 			{
 				Type: prometheus.CounterValue,
 				Desc: prometheus.NewDesc(
@@ -599,6 +611,18 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo
 				},
 				Labels: indexLabels,
 			},
+			{
+				Type: prometheus.GaugeValue,
+				Desc: prometheus.NewDesc(
+					prometheus.BuildFQName(namespace, "index_stats", "index_current"),
+					"The number of documents currently being indexed to an index",
+					indexLabels.keys(), nil,
+				),
+				Value: func(indexStats IndexStatsIndexResponse) float64 {
+					return float64(indexStats.Total.Indexing.IndexCurrent)
+				},
+				Labels: indexLabels,
+			},
 			{
 				Type: prometheus.CounterValue,
 				Desc: prometheus.NewDesc(
@@ -1041,7 +1065,9 @@ func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) {
 	u := *i.url
 	u.Path = path.Join(u.Path, "/_all/_stats")
 	if i.shards {
-		u.RawQuery = "level=shards"
+		u.RawQuery = "ignore_unavailable=true&level=shards"
+	} else {
+		u.RawQuery = "ignore_unavailable=true"
 	}
 
 	res, err := i.client.Get(u.String())

collector/indices_mappings.go

@@ -21,8 +21,8 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

collector/indices_mappings_test.go

@@ -20,7 +20,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestMapping(t *testing.T) {

collector/indices_settings.go

@@ -22,8 +22,8 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -83,7 +83,7 @@ func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL) *I
 					defaultIndicesTotalFieldsLabels, nil,
 				),
 				Value: func(indexSettings Settings) float64 {
-					val, err := strconv.ParseFloat(indexSettings.IndexInfo.Mapping.TotalFields.Limit, 10)
+					val, err := strconv.ParseFloat(indexSettings.IndexInfo.Mapping.TotalFields.Limit, 64)
 					if err != nil {
 						return float64(defaultTotalFieldsValue)
 					}

collector/indices_settings_test.go

@@ -20,7 +20,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestIndicesSettings(t *testing.T) {

collector/indices_test.go

@@ -20,7 +20,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestIndices(t *testing.T) {

collector/nodes.go

@@ -21,8 +21,8 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

collector/nodes_test.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestNodesStats(t *testing.T) {

collector/snapshots.go

@@ -21,8 +21,8 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

collector/snapshots_test.go

@@ -20,7 +20,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/go-kit/kit/log"
+	"github.com/go-kit/log"
 )
 
 func TestSnapshots(t *testing.T) {

examples/grafana/dashboard.json

@@ -2244,7 +2244,7 @@
     ]
   },
   "timezone": "utc",
-  "title": "ElasticSearch Example",
+  "title": "ElasticSearch",
   "uid": "n_nxrE_mk",
-  "version": 1
-}
+  "version": 2
+}

go.mod

@@ -5,19 +5,9 @@ go 1.16
 require (
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
 	github.com/blang/semver v3.5.2-0.20180723201105-3c1074078d32+incompatible
-	github.com/go-kit/kit v0.10.0
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/go-kit/log v0.2.0
 	github.com/imdario/mergo v0.3.12
 	github.com/prometheus/client_golang v1.11.0
-	github.com/prometheus/common v0.29.0
-	github.com/prometheus/procfs v0.7.0 // indirect
-	github.com/prometheus/promu v0.12.0 // indirect
-	go.uber.org/atomic v1.8.0 // indirect
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	github.com/prometheus/common v0.31.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )