Merge 9c69349 into 8f21d49

.gitignore

@@ -0,0 +1,9 @@
+# build artifacts
+.gopath/pkg/
+build/
+
+# test artifacts
+build/cover.out
+build/*.cover.out
+build/cover.html
+*.actual

.travis.yml

@@ -0,0 +1,11 @@
+language: go
+go:
+- 1.13
+
+install:
+- go get -u golang.org/x/lint/golint
+- go get -u github.com/mattn/goveralls
+
+script:
+- make all check GO_BUILDFLAGS="-mod=vendor"
+- $HOME/gopath/bin/goveralls -coverprofile=build/cover.out -service=travis-ci

Makefile

@@ -0,0 +1,67 @@
+PKG     = github.com/sapcc/swift-health-exporter
+PREFIX := /usr
+
+# NOTE: This repo uses Go modules, and uses a synthetic GOPATH at
+# $(CURDIR)/.gopath that is only used for the build cache. $GOPATH/src/ is
+# empty.
+GO            := GOPATH=$(CURDIR)/.gopath GOBIN=$(CURDIR)/build go
+GO_BUILDFLAGS :=
+GO_LDFLAGS    := -s -w
+
+all: build/swift-health-exporter
+
+build/swift-health-exporter: FORCE
+	$(GO) install $(GO_BUILDFLAGS) -ldflags '$(GO_LDFLAGS)' '$(PKG)'
+
+install: FORCE all
+	install -D -m 0755 build/swift-health-exporter "$(DESTDIR)$(PREFIX)/bin/swift-health-exporter"
+
+# which packages to test with static checkers?
+GO_ALLPKGS := $(shell go list $(GO_BUILDFLAGS) $(PKG)/...)
+# which packages to test with `go test`?
+GO_TESTPKGS := $(shell go list $(GO_BUILDFLAGS) -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' $(PKG)/...)
+# which packages to measure coverage for?
+GO_COVERPKGS := $(shell go list $(GO_BUILDFLAGS) $(PKG) $(PKG)/collectors)
+# output files from `go test`
+GO_COVERFILES := $(patsubst %,build/%.cover.out,$(subst /,_,$(GO_TESTPKGS)))
+
+# down below, I need to substitute spaces with commas; because of the syntax,
+# I have to get these separators from variables
+space := $(null) $(null)
+comma := ,
+
+check: FORCE all static-check build/cover.html
+	@printf "\e[1;32m>> All tests successful.\e[0m\n"
+
+static-check: FORCE
+	@if ! hash golint 2>/dev/null; then printf "\e[1;36m>> Installing golint...\e[0m\n"; go get -u golang.org/x/lint/golint; fi
+	@printf "\e[1;36m>> gofmt\e[0m\n"
+	@if s="$$(gofmt -s -l *.go cmd pkg 2>/dev/null)" && test -n "$$s"; then printf ' => %s\n%s\n' gofmt  "$$s"; false; fi
+	@printf "\e[1;36m>> golint\e[0m\n"
+	@if s="$$(golint . && find cmd pkg -type d -exec golint {} \; 2>/dev/null)" && test -n "$$s"; then printf ' => %s\n%s\n' golint "$$s"; false; fi
+	@printf "\e[1;36m>> go vet\e[0m\n"
+	@$(GO) vet $(GO_BUILDFLAGS) $(GO_ALLPKGS)
+
+# detailed unit test run (incl. test coverage)
+build/%.cover.out: FORCE build/mock-swift-dispersion-report build/mock-swift-recon
+	@printf "\e[1;36m>> go test $(subst _,/,$*)\e[0m\n"
+	$(GO) test $(GO_BUILDFLAGS) -ldflags '$(GO_LDFLAGS)' -coverprofile=$@ -covermode=count -coverpkg=$(subst $(space),$(comma),$(GO_COVERPKGS)) $(subst _,/,$*)
+build/cover.out: $(GO_COVERFILES)
+	$(GO) run $(GO_BUILDFLAGS) test/cmd/gocovcat/main.go $(GO_COVERFILES) > $@
+build/cover.html: build/cover.out
+	$(GO) tool cover -html $< -o $@
+
+build/mock-swift-dispersion-report: FORCE
+	$(GO) install $(GO_BUILDFLAGS) -ldflags '$(GO_LDFLAGS)' '$(PKG)/test/cmd/mock-swift-dispersion-report'
+
+build/mock-swift-recon: FORCE
+	$(GO) install $(GO_BUILDFLAGS) -ldflags '$(GO_LDFLAGS)' '$(PKG)/test/cmd/mock-swift-recon'
+
+clean: FORCE
+	rm -rf -- build
+
+vendor: FORCE
+	$(GO) mod tidy
+	$(GO) mod vendor
+
+.PHONY: FORCE

README.md

@@ -0,0 +1,96 @@
+# Swift health exporter
+
+[![Build Status](https://travis-ci.org/sapcc/swift-health-exporter.svg?branch=master)](https://travis-ci.org/sapcc/swift-health-exporter)
+[![Coverage Status](https://coveralls.io/repos/github/sapcc/swift-health-exporter/badge.svg?branch=master)](https://coveralls.io/github/sapcc/swift-health-exporter?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/sapcc/swift-health-exporter)](https://goreportcard.com/report/github.com/sapcc/swift-health-exporter)
+
+This exporter uses the `swift-dispersion-report` and `swift-recon` tools to
+emit Prometheus metrics about the health of an OpenStack Swift cluster.
+
+The exporter has been tested with OpenStack Swift Train (v2.23.0) and above.
+For older versions of Swift, you might want to look at
+[swift-health-statsd](https://github.com/sapcc/swift-health-statsd).
+
+## Installation
+
+The only required build dependency is [Go](https://golang.org/):
+
+```sh
+make install
+```
+
+This will install the binary in `/usr/bin/`.
+
+## Usage
+
+`swift-health-exporter` will find the executables for `swift-recon` and
+`swift-dispersion-report` in the directories named by the `$PATH` environment
+variable.
+
+To start collecting metrics, you simply need to:
+
+```sh
+swift-health-exporter
+```
+
+If the `swift-recon` and `swift-dispersion-report` are not in the directories
+named by the `$PATH` then you **must** provide the respective paths to
+these executables using the configuration options.
+
+### Configuration options
+
+The following environment variables are recognized:
+
+| Variable                       | Required                          | Description                                                                               |
+| ----------------               | ----------------                  | ----------------                                                                          |
+| `SWIFT_DISPERSION_REPORT_PATH` | yes, if executable not in `$PATH` | Path to the `swift-dispersion-report` executable.                                         |
+| `SWIFT_RECON_PATH`             | yes, if executable not in `$PATH` | Path to the `swift-recon` executable.                                                     |
+| `DEBUG`                        | no                                | If this option is set to `true` then `swift-health-exporter` will also output debug logs. |
+
+## Metrics
+
+Metric are exposed at port `9520`. This port has been
+[allocated](https://github.com/prometheus/prometheus/wiki/Default-port-allocations)
+for `swift-health-exporter`.
+
+### Dispersion Report
+
+| Metric                                       | Labels      |
+| ----------------                             | ----------- |
+| `swift_dispersion_container_copies_expected` |             |
+| `swift_dispersion_container_copies_found`    |             |
+| `swift_dispersion_container_copies_missing`  |             |
+| `swift_dispersion_container_overlapping`     |             |
+| `swift_dispersion_object_copies_expected`    |             |
+| `swift_dispersion_object_copies_found`       |             |
+| `swift_dispersion_object_copies_missing`     |             |
+| `swift_dispersion_object_overlapping`        |             |
+| `swift_dispersion_exit_code`                 | `query`     |
+
+### Recon
+
+| Metric                                          | Labels               |
+| ----------------                                | -----------          |
+| `swift_cluster_accounts_quarantined`            | `storage_ip`         |
+| `swift_cluster_accounts_replication_age`        | `storage_ip`         |
+| `swift_cluster_accounts_replication_duration`   | `storage_ip`         |
+| `swift_cluster_containers_quarantined`          | `storage_ip`         |
+| `swift_cluster_containers_replication_age`      | `storage_ip`         |
+| `swift_cluster_containers_replication_duration` | `storage_ip`         |
+| `swift_cluster_containers_updater_sweep_time`   | `storage_ip`         |
+| `swift_cluster_drives_audit_errors`             | `storage_ip`         |
+| `swift_cluster_drives_unmounted`                | `storage_ip`         |
+| `swift_cluster_md5_all`                         | `kind`               |
+| `swift_cluster_md5_errors`                      | `kind`               |
+| `swift_cluster_md5_matched`                     | `kind`               |
+| `swift_cluster_md5_not_matched`                 | `kind`               |
+| `swift_cluster_objects_quarantined`             | `storage_ip`         |
+| `swift_cluster_objects_replication_age`         | `storage_ip`         |
+| `swift_cluster_objects_replication_duration`    | `storage_ip`         |
+| `swift_cluster_objects_updater_sweep_time`      | `storage_ip`         |
+| `swift_cluster_storage_capacity_bytes`          | `storage_ip`         |
+| `swift_cluster_storage_free_bytes`              | `storage_ip`         |
+| `swift_cluster_storage_used_bytes`              | `storage_ip`         |
+| `swift_cluster_storage_used_percent_by_disk`    | `storage_ip`, `disk` |
+| `swift_cluster_storage_used_percent`            | `storage_ip`         |
+| `swift_recon_exit_code`                         | `query`              |

collectors/dispersion.go

@@ -0,0 +1,173 @@
+// Copyright 2019 SAP SE
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"encoding/json"
+	"os/exec"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sapcc/go-bits/logg"
+)
+
+var (
+	dispersionCntrCopiesExpectedDesc = prometheus.NewDesc(
+		"swift_dispersion_container_copies_expected",
+		"Expected container copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionCntrCopiesFoundDesc = prometheus.NewDesc(
+		"swift_dispersion_container_copies_found",
+		"Found container copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionCntrCopiesMissingDesc = prometheus.NewDesc(
+		"swift_dispersion_container_copies_missing",
+		"Missing container copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionCntrOverlappingDesc = prometheus.NewDesc(
+		"swift_dispersion_container_overlapping",
+		"Expected container copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+
+	dispersionObjCopiesExpectedDesc = prometheus.NewDesc(
+		"swift_dispersion_object_copies_expected",
+		"Expected object copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionObjCopiesFoundDesc = prometheus.NewDesc(
+		"swift_dispersion_object_copies_found",
+		"Found object copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionObjCopiesMissingDesc = prometheus.NewDesc(
+		"swift_dispersion_object_copies_missing",
+		"Missing object copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+	dispersionObjOverlappingDesc = prometheus.NewDesc(
+		"swift_dispersion_object_overlapping",
+		"Expected object copies reported by the swift-dispersion-report tool.",
+		nil, nil,
+	)
+
+	dispersionTaskErrorDesc = prometheus.NewDesc(
+		"swift_dispersion_exit_code",
+		"The exit code for a Swift Dispersion Report query execution.",
+		[]string{"query"}, nil,
+	)
+)
+
+// DispersionCollector implements the prometheus.Collector interface.
+type DispersionCollector struct {
+	pathToExecutable string
+}
+
+// NewDispersionCollector creates a new DispersionCollector.
+func NewDispersionCollector(pathToExecutable string) *DispersionCollector {
+	return &DispersionCollector{
+		pathToExecutable: pathToExecutable,
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c *DispersionCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c *DispersionCollector) Collect(ch chan<- prometheus.Metric) {
+	errCount := 0
+	query := "--dump-json"
+	defer func() {
+		ch <- prometheus.MustNewConstMetric(
+			dispersionTaskErrorDesc,
+			prometheus.CounterValue, float64(errCount),
+			query,
+		)
+	}()
+
+	var dispersionReport struct {
+		Object struct {
+			Expected    int64 `json:"copies_expected"`
+			Found       int64 `json:"copies_found"`
+			Overlapping int64 `json:"overlapping"`
+			Missing     int64
+		} `json:"object"`
+		Container struct {
+			Expected    int64 `json:"copies_expected"`
+			Found       int64 `json:"copies_found"`
+			Overlapping int64 `json:"overlapping"`
+			Missing     int64
+		} `json:"container"`
+	}
+
+	out, err := exec.Command(c.pathToExecutable, query).Output()
+	if err != nil {
+		logg.Error("swift-dispersion-report: %v", err)
+		errCount++
+		return
+	}
+	err = json.Unmarshal(out, &dispersionReport)
+	if err != nil {
+		logg.Error("swift-dispersion-report: %v", err)
+		errCount++
+		return
+	}
+
+	cntr := dispersionReport.Container
+	if cntr.Expected > 0 && cntr.Found > 0 {
+		cntr.Missing = cntr.Expected - cntr.Found
+	}
+	ch <- prometheus.MustNewConstMetric(
+		dispersionCntrCopiesExpectedDesc,
+		prometheus.GaugeValue, float64(cntr.Expected),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionCntrCopiesFoundDesc,
+		prometheus.GaugeValue, float64(cntr.Found),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionCntrCopiesMissingDesc,
+		prometheus.GaugeValue, float64(cntr.Missing),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionCntrOverlappingDesc,
+		prometheus.GaugeValue, float64(cntr.Overlapping),
+	)
+
+	obj := dispersionReport.Object
+	if obj.Expected > 0 && obj.Found > 0 {
+		obj.Missing = obj.Expected - obj.Found
+	}
+	ch <- prometheus.MustNewConstMetric(
+		dispersionObjCopiesExpectedDesc,
+		prometheus.GaugeValue, float64(obj.Expected),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionObjCopiesFoundDesc,
+		prometheus.GaugeValue, float64(obj.Found),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionObjCopiesMissingDesc,
+		prometheus.GaugeValue, float64(obj.Missing),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		dispersionObjOverlappingDesc,
+		prometheus.GaugeValue, float64(obj.Overlapping),
+	)
+}