Merge pull request #1 from improbable-eng/master

backmerging

CHANGELOG.md

@@ -11,6 +11,23 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ## Unreleased
 
+## [v0.3.2](https://github.com/improbable-eng/thanos/releases/tag/v0.3.2) - 2019.03.04
+
+### Added
+- [#851](https://github.com/improbable-eng/thanos/pull/851) New read API endpoint for api/v1/rules and api/v1/alerts.
+- [#873](https://github.com/improbable-eng/thanos/pull/873) Store: fix set index cache LRU.
+
+### Fixed
+- [#833](https://github.com/improbable-eng/thanos/issues/833) Store Gateway matcher regression for intersecting with empty posting.
+- [#867](https://github.com/improbable-eng/thanos/pull/867) Fixed race condition in sidecare between reloader and shipper.
+
+## [v0.3.1](https://github.com/improbable-eng/thanos/releases/tag/v0.3.1) - 2019.02.18
+
+### Fixed
+- [#829](https://github.com/improbable-eng/thanos/issues/829) Store Gateway crashing due to `slice bounds out of range`.
+- [#834](https://github.com/improbable-eng/thanos/issues/834) Store Gateway matcher regression for `<>` `!=`.
+
+
 ## [v0.3.0](https://github.com/improbable-eng/thanos/releases/tag/v0.3.0) - 2019.02.08
 
 ### Added

Dockerfile.multi-stage

@@ -0,0 +1,17 @@
+FROM golang:1.11-alpine3.9 as builder
+
+ADD . $GOPATH/src/github.com/improbable-eng/thanos
+WORKDIR $GOPATH/src/github.com/improbable-eng/thanos
+
+RUN apk update && apk upgrade && apk add --no-cache alpine-sdk
+
+RUN git update-index --refresh; make
+
+# -----------------------------------------------------------------------------
+
+FROM quay.io/prometheus/busybox:latest
+LABEL maintainer="The Thanos Authors"
+
+COPY --from=builder /go/src/github.com/improbable-eng/thanos/thanos /bin/thanos
+
+ENTRYPOINT [ "/bin/thanos" ]

Gopkg.toml

@@ -26,7 +26,7 @@ ignored = ["github.com/improbable-eng/thanos/benchmark/*"]
 
 [[constraint]]
   name = "github.com/pkg/errors"
-  version = "0.8.0"
+  version = "0.8.1"
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"

Makefile

@@ -99,6 +99,12 @@ docker: build
 	@echo ">> building docker image '${DOCKER_IMAGE_NAME}'"
 	@docker build -t "${DOCKER_IMAGE_NAME}" .
 
+#docker-multi-stage builds docker image using multi-stage.
+.PHONY: docker-multi-stage
+docker-multi-stage:
+	@echo ">> building docker image '${DOCKER_IMAGE_NAME}' with Dockerfile.multi-stage"
+	@docker build -f Dockerfile.multi-stage -t "${DOCKER_IMAGE_NAME}" .
+
 # docker-push pushes docker image build under `${DOCKER_IMAGE_NAME}` to improbable/"$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)"
 .PHONY: docker-push
 docker-push:

README.md

@@ -4,6 +4,7 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/improbable-eng/thanos)](https://goreportcard.com/report/github.com/improbable-eng/thanos)
 [![GoDoc](https://godoc.org/github.com/improbable-eng/thanos?status.svg)](https://godoc.org/github.com/improbable-eng/thanos)
 [![Slack](https://img.shields.io/badge/join%20slack-%23thanos-brightgreen.svg)](https://join.slack.com/t/improbable-eng/shared_invite/enQtMzQ1ODcyMzQ5MjM4LWY5ZWZmNGM2ODc5MmViNmQ3ZTA3ZTY3NzQwOTBlMTkzZmIxZTIxODk0OWU3YjZhNWVlNDU3MDlkZGViZjhkMjc)
+[![Docker Pulls](https://img.shields.io/docker/pulls/improbable/thanos.svg?maxAge=604800)](https://hub.docker.com/r/improbable/thanos/)
 
 ## Overview
 

VERSION

@@ -1 +1 @@
-0.3.0-master
+0.3.2-master

benchmark/cmd/thanosbench/resources.go

@@ -12,6 +12,7 @@ import (
 	prom "github.com/prometheus/prometheus/config"
 	"gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -207,7 +208,7 @@ func createPrometheus(opts *opts, name string, bucket string) *appsv1.StatefulSe
 		Name:      name,
 		Namespace: promNamespace,
 		Labels: map[string]string{
-			"app": name,
+			"app":                  name,
 			"thanos-gossip-member": "true",
 		},
 	}
@@ -370,7 +371,7 @@ func createThanosQuery(opts *opts) (*v1.Service, *v1.Pod) {
 		Name:      "thanos-query",
 		Namespace: thanosNamespace,
 		Labels: map[string]string{
-			"app": "thanos-query",
+			"app":                  "thanos-query",
 			"thanos-gossip-member": "true",
 		},
 	}

cmd/thanos/compact.go

@@ -92,6 +92,9 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri
 	maxCompactionLevel := cmd.Flag("debug.max-compaction-level", fmt.Sprintf("Maximum compaction level, default is %d: %s", compactions.maxLevel(), compactions.String())).
 		Hidden().Default(strconv.Itoa(compactions.maxLevel())).Int()
 
+	blockSyncConcurrency := cmd.Flag("block-sync-concurrency", "Number of goroutines to use when syncing block metadata from object storage.").
+		Default("20").Int()
+
 	m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
 		return runCompact(g, logger, reg,
 			*httpAddr,
@@ -108,6 +111,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri
 			name,
 			*disableDownsampling,
 			*maxCompactionLevel,
+			*blockSyncConcurrency,
 		)
 	}
 }
@@ -126,6 +130,7 @@ func runCompact(
 	component string,
 	disableDownsampling bool,
 	maxCompactionLevel int,
+	blockSyncConcurrency int,
 ) error {
 	halted := prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "thanos_compactor_halted",
@@ -157,7 +162,7 @@ func runCompact(
 		}
 	}()
 
-	sy, err := compact.NewSyncer(logger, reg, bkt, syncDelay)
+	sy, err := compact.NewSyncer(logger, reg, bkt, syncDelay, blockSyncConcurrency)
 	if err != nil {
 		return errors.Wrap(err, "create syncer")
 	}

cmd/thanos/downsample.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"github.com/improbable-eng/thanos/pkg/component"
 	"os"
 	"path"
 	"path/filepath"
@@ -14,6 +13,7 @@ import (
 	"github.com/improbable-eng/thanos/pkg/block"
 	"github.com/improbable-eng/thanos/pkg/block/metadata"
 	"github.com/improbable-eng/thanos/pkg/compact/downsample"
+	"github.com/improbable-eng/thanos/pkg/component"
 	"github.com/improbable-eng/thanos/pkg/objstore"
 	"github.com/improbable-eng/thanos/pkg/objstore/client"
 	"github.com/improbable-eng/thanos/pkg/runutil"

cmd/thanos/rule.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -29,6 +30,7 @@ import (
 	"github.com/improbable-eng/thanos/pkg/extprom"
 	"github.com/improbable-eng/thanos/pkg/objstore/client"
 	"github.com/improbable-eng/thanos/pkg/promclient"
+	"github.com/improbable-eng/thanos/pkg/rule/api"
 	"github.com/improbable-eng/thanos/pkg/runutil"
 	"github.com/improbable-eng/thanos/pkg/shipper"
 	"github.com/improbable-eng/thanos/pkg/store"
@@ -565,6 +567,9 @@ func runRule(
 
 		ui.NewRuleUI(logger, mgr, alertQueryURL.String(), flagsMap).Register(router.WithPrefix(webRoutePrefix))
 
+		api := v1.NewAPI(logger, mgr)
+		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger)
+
 		mux := http.NewServeMux()
 		registerMetrics(mux, reg)
 		registerProfile(mux)

cmd/thanos/sidecar.go

@@ -256,19 +256,19 @@ func runSidecar(
 		}
 
 		ctx, cancel := context.WithCancel(context.Background())
-		var s *shipper.Shipper
-		if uploadCompacted {
-			s, err = shipper.NewWithCompacted(ctx, logger, reg, dataDir, bkt, m.Labels, metadata.SidecarSource, promURL)
-			if err != nil {
-				return errors.Wrap(err, "create shipper")
-			}
-		} else {
-			s = shipper.New(logger, reg, dataDir, bkt, m.Labels, metadata.SidecarSource)
-		}
-
 		g.Add(func() error {
 			defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
 
+			var s *shipper.Shipper
+			if uploadCompacted {
+				s, err = shipper.NewWithCompacted(ctx, logger, reg, dataDir, bkt, m.Labels, metadata.SidecarSource, m.promURL)
+				if err != nil {
+					return errors.Wrap(err, "create shipper")
+				}
+			} else {
+				s = shipper.New(logger, reg, dataDir, bkt, m.Labels, metadata.SidecarSource)
+			}
+
 			return runutil.Repeat(30*time.Second, ctx.Done(), func() error {
 				if uploaded, err := s.Sync(ctx); err != nil {
 					level.Warn(logger).Log("err", err, "uploaded", uploaded)

docs/components/compact.md

@@ -67,5 +67,8 @@ Flags:
                            in bucket. 0d - disables this retention
   -w, --wait               Do not exit after all compactions have been processed
                            and wait for new work.
+      --block-sync-concurrency=20  
+                           Number of goroutines to use when syncing block
+                           metadata from object storage.
 
 ```

docs/components/sidecar.md

@@ -13,7 +13,14 @@ Prometheus servers connected to the Thanos cluster via the sidecar are subject t
 The retention is recommended to not be lower than three times the block duration. This achieves resilience in the face of connectivity issues 
 to the object storage since all local data will remain available within the Thanos cluster. If connectivity gets restored the backlog of blocks gets uploaded to the object storage.
 
+```console
+$ prometheus \
+  --storage.tsdb.max-block-duration=2h \
+  --storage.tsdb.min-block-duration=2h \
+  --web.enable-lifecycle
 ```
+
+```console
 $ thanos sidecar \
     --tsdb.path        "/path/to/prometheus/data/dir" \
     --prometheus.url   "http://localhost:9090" \

docs/getting_started.md

@@ -59,8 +59,8 @@ Rolling this out has little to zero impact on the running Prometheus instance. I
 
 If you are not interested in backing up any data, the `--objstore.config-file` flag can simply be omitted.
 
-* _[Example Kubernetes manifest](../kube/manifests/prometheus.yaml)_
-* _[Example Kubernetes manifest with GCS upload](../kube/manifests/prometheus-gcs.yaml)_
+* _[Example Kubernetes manifest](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_
+* _[Example Kubernetes manifest with Minio upload](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_
 * _[Details & Config for other object stores](./storage.md)_
 
 ### [Store API](/pkg/store/storepb/rpc.proto#L19)
@@ -78,8 +78,8 @@ thanos sidecar \
     --grpc-address              0.0.0.0:19090              # GRPC endpoint for StoreAPI
 ```
 
-* _[Example Kubernetes manifest](../kube/manifests/prometheus.yaml)_
-* _[Example Kubernetes manifest with GCS upload](../kube/manifests/prometheus-gcs.yaml)_
+* _[Example Kubernetes manifest](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_
+* _[Example Kubernetes manifest with GCS upload](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_
 
 ### External Labels
 
@@ -142,7 +142,7 @@ thanos query \
 
 Go to the configured HTTP address, and you should now be able to query across all Prometheus instances and receive de-duplicated data.
 
-* _[Example Kubernetes manifest](../kube/manifests/thanos-query.yaml)_
+* _[Example Kubernetes manifest](../tutorials/kubernetes-demo/manifests/thanos-querier.yaml)_
 
 ## Communication Between Components
 
@@ -218,8 +218,8 @@ When to use gossip vs store flags?
 Configuration of initial peers is flexible and the argument can be repeated for Thanos to try different approaches.
 Additional flags for cluster configuration exist but are typically not needed. Check the `--help` output for further information.
 
-* _[Example Kubernetes manifest](../kube/manifests/prometheus.yaml)_
-* _[Example Kubernetes manifest with GCS upload](../kube/manifests/prometheus-gcs.yaml)_
+* _[Example Kubernetes manifest](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml)_
+* _[Example Kubernetes manifest with GCS upload](../tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml)_
 
 ## [Store Gateway](components/store.md)
 
@@ -237,7 +237,7 @@ thanos store \
 
 The store gateway occupies small amounts of disk space for caching basic information about data in the object storage. This will rarely exceed more than a few gigabytes and is used to improve restart times. It is not useful but not required to preserve it across restarts.
 
-* _[Example Kubernetes manifest](../kube/manifests/thanos-store.yaml)_
+* _[Example Kubernetes manifest](../tutorials/kubernetes-demo/manifests/thanos-store-gateway.yaml)_
 
 ## [Compactor](components/compact.md)
 
@@ -263,7 +263,7 @@ which does rule and alert evaluation on top of given Thanos Querier.
 
 # All-in-one example
 
-You can find one-box example with minikube [here](../kube/README.md).
+You can find kubernetes manifests [here](../tutorials/kubernetes-demo/manifests).
 
 # Dashboards