Merge remote-tracking branch 'origin/master' into add_hash_method

Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>

.circleci/config.yml

@@ -1,17 +1,18 @@
 # NOTE: Current plan gives 1500 build minutes per month.
-version: 2
-# https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
-defaults: &defaults
-  docker:
-    # Built by Thanos make docker-ci
-    - image: &default-docker-image quay.io/thanos/thanos-ci:v1.2-go1.15-node
+version: 2.1
 
-jobs:
-  test:
+executors:
+  golang:
     docker:
-      - image: *default-docker-image
+      - image: cimg/go:1.15-node
+  golang-test:
+    docker:
+      - image: cimg/go:1.15-node
       - image: beaukode/docker-swift-onlyone-authv2-keystone
-    working_directory: /go/src/github.com/thanos-io/thanos
+
+jobs:
+  test:
+    executor: golang-test
     environment:
       GO111MODULE: 'on'
     steps:
@@ -30,7 +31,6 @@ jobs:
       - run:
           name: "Run unit tests."
           environment:
-            GOBIN: "/go/bin"
             THANOS_TEST_OBJSTORE_SKIP: AZURE,COS,ALIYUNOSS
             # Variables for Swift testing.
             OS_AUTH_URL: http://127.0.0.1:5000/v2.0
@@ -65,10 +65,7 @@ jobs:
             - .build
 
   publish_master:
-    <<: *defaults
-    working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
+    executor: golang
     steps:
       - checkout
       - setup_remote_docker:
@@ -85,10 +82,7 @@ jobs:
       - run: make docker-push
 
   publish_release:
-    <<: *defaults
-    working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
+    executor: golang
     steps:
       - checkout
       - setup_remote_docker:

.gitpod.yml

@@ -0,0 +1,13 @@
+tasks:
+  - init: >
+      test -f go.mod && go get -v ./...;
+      wget https://github.com/prometheus/prometheus/releases/download/v2.22.2/prometheus-2.22.2.linux-amd64.tar.gz --directory-prefix=/tmp;
+      cd /tmp && tar xvfz prometheus-*.tar.gz && rm prometheus-*.tar.gz;
+      mv prometheus-*/prometheus /workspace/go/bin/;
+      wget https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && chmod u+x ./wait-for-it.sh && cd /workspace/thanos/scripts && mv /tmp/wait-for-it.sh .;
+      ./quickstart.sh;
+      for i in {0..2};do ./wait-for-it.sh -q localhost:909"$i" --  && echo "Prometheus instance no.$((i+1)) localhost:909$i is up"; done;
+      ./wait-for-it.sh localhost:10914 --  && echo "Thanos frontend at localhost:10914 is up";
+ports:
+  - port: 10914
+    onOpen: open-preview

.promu.yml

@@ -1,5 +1,5 @@
 go:
-  version: 1.15.0
+  version: 1.15
 repository:
   path: github.com/thanos-io/thanos
 build:

CHANGELOG.md

@@ -1,3 +1,4 @@
+
 # Changelog
 
 All notable changes to this project will be documented in this file.
@@ -15,14 +16,15 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 
 - [#3469](https://github.com/thanos-io/thanos/pull/3469) StoreAPI: Added `hints` field to `LabelNamesRequest` and `LabelValuesRequest`. Hints in an opaque data structure that can be used to carry additional information from the store and its content is implementation specific.
 - [#3031](https://github.com/thanos-io/thanos/pull/3031) Compact/Sidecar/other writers: added `--hash-func`. If some function has been specified, writers calculate hashes using that function of each file in a block before uploading them. If those hashes exist in the `meta.json` file then Compact does a smart thing and does not download the files if they already exist on disk. This also means that the data directory passed to Thanos Compact is only *cleared once at boot* or *if everything succeeds*. So, if you, for example, use persistent volumes on k8s and your Thanos Compact crashes or fails to make an iteration properly then the last downloaded files are not wiped from the disk. The directories that were created the last time are only wiped again after a successful iteration or if the previously picked up blocks have disappeared.
+- [#3421](https://github.com/thanos-io/thanos/pull/3421) Tools: Added `thanos tools bucket rewrite` command allowing to delete series from given block.
 
 ### Fixed
 
--
+- [#3527](https://github.com/thanos-io/thanos/pull/3527) Query Frontend: Fix query_range behavior when start/end times are the same
 
 ### Changed
 
--
+- [#3496](https://github.com/thanos-io/thanos/pull/3496) s3: Respect SignatureV2 flag for all credential providers.
 
 ## [v0.17.0](https://github.com/thanos-io/thanos/releases/tag/v0.17.0) - 2020.11.18
 

CONTRIBUTING.md

@@ -2,37 +2,37 @@
 
 This document explain the process of contributing to the Thanos project.
 
-First of all please follow the [CODE_OF_CONDUCT](/CODE_OF_CONDUCT.md) in all your interactions with the project.
+First of all please follow the [CODE_OF_CONDUCT](/CODE_OF_CONDUCT.md) in all your interactions within the project.
 
 ## Thanos Philosophy
 
-The philosophy of Thanos and our community is borrowing much from UNIX philosophy and the golang programming language.
+The philosophy of Thanos and our community borrows heavily from UNIX philosophy and the Golang programming language.
 
-* Each sub command should do one thing and do it well
+* Each sub command should do one thing and do it well.
   * eg. thanos query proxies incoming calls to known store API endpoints merging the result
-* Write components that work together
+* Write components that work together.
   * e.g. blocks should be stored in native prometheus format
-* Make it easy to read, write, and, run components
+* Make it easy to read, write, and run components.
   * e.g. reduce complexity in system design and implementation
 
 ## Feedback / Issues
 
 If you encounter any issue or you have an idea to improve, please:
 
 * Search through Google and [existing open and closed GitHub Issues](https://github.com/thanos-io/thanos/issues) for the
-answer first. If you find relevant topic, please comment on the issue.
-* If not found, please add an issue to [GitHub issues](https://github.com/thanos-io/thanos/issues). Please provide
-all relevant information as template suggest.
+answer first. If you find a relevant topic, please comment on the issue.
+* If none of the issues are relevant, please add an issue to [GitHub issues](https://github.com/thanos-io/thanos/issues). Please provide
+any relevant information as suggested by the Pull Request template.
 * If you have a quick question you might want to also ask on #thanos or #thanos-dev slack channel in the CNCF workspace.
-We are recommending, using GitHub issues for issues and feedback, because GitHub issues are track-able.
+We recommend using GitHub issues for issues and feedback, because GitHub issues are trackable.
 
-If you encounter security vulnerability, please refer to [Reporting a Vulnerability process](/SECURITY.md)
+If you encounter a security vulnerability, please refer to [Reporting a Vulnerability process](/SECURITY.md#reporting-a-vulnerability)
 
 ## Adding New Features / Components
 
-When contributing not obvious change to Thanos repository, please first
-discuss the change you wish to make via issue or slack, or any other
-method with the owners of this repository before making a change.
+When contributing a complex change to Thanos repository, please
+discuss the change you wish to make within a Github issue, in Slack, or by another
+method with the owners of this repository before making the change.
 
 Adding a large new feature or/and component to Thanos should be done by first creating a [proposal](/docs/proposals) document outlining the design decisions of the change, motivations for the change, and any alternatives that might have been considered.
 
@@ -46,9 +46,9 @@ In the code and documentation prefer non-offensive terminology, for example:
 
 ## Components Naming
 
-Thanos is a distributed system composed with several services and CLI tools as listed [here](/cmd/thanos).
+Thanos is a distributed system comprised of several services and CLI tools as listed [here](/cmd/thanos).
 
-When we refer to them as technical reference we use verb form: `store`, `compact`, `rule`, `query`, `query_frontend`. This includes:
+When we refer to them in a technical capacity we use the verbal form: `store`, `compact`, `rule`, `query`, `query_frontend`. This includes:
 
 * Code
 * Metrics
@@ -59,7 +59,7 @@ When we refer to them as technical reference we use verb form: `store`, `compact
 * Package names
 * Log messages, traces
 
-However, when speaking about those or explaining we use `actor` noun form: `store gateway`, `compactor`, `ruler`, `querier`, `query frontend`. This includes areas like:
+However, when discussing these components in a more general manner we use the `actor` noun form: `store gateway`, `compactor`, `ruler`, `querier`, `query frontend`. This includes areas like:
 
 * Public communication
 * Documentation
@@ -73,17 +73,19 @@ The following section explains various suggestions and procedures to note during
 
 * It is strongly recommended that you use Linux distributions systems or OSX for development.
 * Go 1.13.9 or newer installed.
-* For React UI, you will need a working NodeJS environment and the Yarn package manager to compile the Web UI assets
+* For React UI, you will need a working NodeJS environment and the Yarn package manager to compile the Web UI assets.
 
 ### First Steps
 
-It's key to get familiarized with style guide and mechanics of Thanos, especially if your contribution touches more than one
+It's key to get familiarized with the style guide and mechanics of Thanos, especially if your contribution touches more than one
 component of the Thanos distributed system. We recommend:
 
 * Reading the [getting started docs](docs/getting-started.md) and working through them, or alternatively working through the [Thanos tutorial](https://katacoda.com/thanos).
 * Familiarizing yourself with our [coding style guidelines.](docs/contributing/coding-style-guide.md).
 * Familiarizing yourself with the [Makefile](Makefile) commands, for example `format`, `build`, `proto`, `docker` and `test`.
-`make help` will print most of available commands with details.
+`make help` will print most of available commands with relevant details.
+* Spin up a prebuilt dev environment using Gitpod.io
+[![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos)
 
 ### Pull Request Process
 

Makefile

@@ -257,14 +257,6 @@ install-deps: ## Installs dependencies for integration tests. It installs suppor
 install-deps: $(ALERTMANAGER) $(MINIO) $(PROMETHEUS_ARRAY)
 	@echo ">>GOBIN=$(GOBIN)"
 
-.PHONY: docker-ci
-docker-ci: ## Builds and pushes docker image used by our CI. This is done to cache our tools and dependencies. To be run by Thanos maintainer.
-docker-ci: install-deps
-	@docker build -t thanos-ci -f Dockerfile.thanos-ci .
-	@echo ">> pushing thanos-ci image"
-	@docker tag "thanos-ci" "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
-	@docker push "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
-
 .PHONY: check-git
 check-git:
 ifneq ($(GIT),)
@@ -310,7 +302,6 @@ github.com/prometheus/client_golang/prometheus.{DefaultGatherer,DefBuckets,NewUn
 github.com/prometheus/client_golang/prometheus.{NewCounter,NewCounterVec,NewCounterVec,NewGauge,NewGaugeVec,NewGaugeFunc,\
 NewHistorgram,NewHistogramVec,NewSummary,NewSummaryVec}=github.com/prometheus/client_golang/prometheus/promauto.{NewCounter,\
 NewCounterVec,NewCounterVec,NewGauge,NewGaugeVec,NewGaugeFunc,NewHistorgram,NewHistogramVec,NewSummary,NewSummaryVec},\
-github.com/prometheus/prometheus/tsdb/errors=github.com/thanos-io/thanos/pkg/errutil,\
 sync/atomic=go.uber.org/atomic" ./...
 	@$(FAILLINT) -paths "fmt.{Print,Println,Sprint}" -ignore-tests ./...
 	@echo ">> linting all of the Go files GOGC=${GOGC}"

README.md

@@ -12,6 +12,7 @@
 [![go](https://github.com/thanos-io/thanos/workflows/go/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Ago)
 [![react](https://github.com/thanos-io/thanos/workflows/react/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Areact)
 [![docs](https://github.com/thanos-io/thanos/workflows/docs/badge.svg)](https://github.com/thanos-io/thanos/actions?query=workflow%3Adocs)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/thanos-io/thanos)
 
 ## Overview
 

SECURITY.md

@@ -1,18 +1,18 @@
 # Security Policy
 
-As the Thanos team we are not security experts.
+At the Thanos team we are not security experts.
 However we try our best to avoid security concerns and to avoid writing features that handle sensitive information at all.
 
 It's worth noting that we assume metric data to be sensitive and important.
-External labels and query API parameters are considered less sensitive, as they are logged and put into metric/traces.
+External labels and query API parameters are considered less sensitive, as they are logged and put into metrics/traces.
 
 ## What You CAN Expect:
 
 * We follow best programming practices.
 We test heavily, including e2e tests against major object storages.
 We use vetting and static analysis tools on every pull request.
-We use secure protocols for building process, e.g. when producing Docker images.
-* We don't put any data that is stored in TSDB into logs or instrumentation .
+We use secure protocols for building processes, e.g. when producing Docker images.
+* We don't put any data that is stored in the TSDB into logs or instrumentation.
 * If we use crypto tools, we always rely on FLOSS and standard libraries, like the official [Go crypt](https://golang.org/pkg/crypto/)
   library.
 * We always use TLS by default for communication with all object storages.
@@ -31,7 +31,7 @@ We recommend setting server-side encryption for object storage.
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 0.10.1   | :white_check_mark: |
+| >= 0.10.1  | :white_check_mark: |
 | < 0.10.1   | :x:                |
 
 ## Reporting a Vulnerability

cmd/thanos/compact.go

@@ -182,7 +182,7 @@ func runCompact(
 		return errors.Wrap(err, "get content of relabel configuration")
 	}
 
-	relabelConfig, err := block.ParseRelabelConfig(relabelContentYaml)
+	relabelConfig, err := block.ParseRelabelConfig(relabelContentYaml, block.SelectorSupportedRelabelActions)
 	if err != nil {
 		return err
 	}
@@ -197,21 +197,24 @@ func runCompact(
 	// While fetching blocks, we filter out blocks that were marked for deletion by using IgnoreDeletionMarkFilter.
 	// The delay of deleteDelay/2 is added to ensure we fetch blocks that are meant to be deleted but do not have a replacement yet.
 	// This is to make sure compactor will not accidentally perform compactions with gap instead.
-	ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, deleteDelay/2)
+	ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, deleteDelay/2, fetcherConcurrency)
 	duplicateBlocksFilter := block.NewDeduplicateFilter()
 
-	baseMetaFetcher, err := block.NewBaseFetcher(logger, 32, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg))
+	baseMetaFetcher, err := block.NewBaseFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg))
 	if err != nil {
 		return errors.Wrap(err, "create meta fetcher")
 	}
 
-	enableVerticalCompaction := false
+	enableVerticalCompaction := conf.enableVerticalCompaction
 	if len(conf.dedupReplicaLabels) > 0 {
 		enableVerticalCompaction = true
 		level.Info(logger).Log(
-			"msg", "deduplication.replica-label specified, vertical compaction is enabled",
-			"dedupReplicaLabels",
-			strings.Join(conf.dedupReplicaLabels, ","),
+			"msg", "deduplication.replica-label specified, enabling vertical compaction", "dedupReplicaLabels", strings.Join(conf.dedupReplicaLabels, ","),
+		)
+	}
+	if enableVerticalCompaction {
+		level.Info(logger).Log(
+			"msg", "vertical compaction is enabled", "compact.enable-vertical-compaction", fmt.Sprintf("%v", conf.enableVerticalCompaction),
 		)
 	}
 
@@ -555,6 +558,7 @@ type compactConfig struct {
 	label                                          string
 	maxBlockIndexSize                              units.Base2Bytes
 	hashFunc                                       string
+	enableVerticalCompaction                       bool
 }
 
 func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) {
@@ -611,6 +615,11 @@ func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) {
 		"or compactor is ignoring the deletion because it's compacting the block at the same time.").
 		Default("48h").SetValue(&cc.deleteDelay)
 
+	cmd.Flag("compact.enable-vertical-compaction", "Experimental. When set to true, compactor will allow overlaps and perform **irreversible** vertical compaction. See https://thanos.io/tip/components/compact.md/#vertical-compactions to read more."+
+		"Please note that this uses a NAIVE algorithm for merging (no smart replica deduplication, just chaining samples together)."+
+		"NOTE: This flag is ignored and (enabled) when --deduplication.replica-label flag is set.").
+		Hidden().Default("false").BoolVar(&cc.enableVerticalCompaction)
+
 	cmd.Flag("deduplication.replica-label", "Label to treat as a replica indicator of blocks that can be deduplicated (repeated flag). This will merge multiple replica blocks into one. This process is irreversible."+
 		"Experimental. When it is set to true, compactor will ignore the given labels so that vertical compaction can merge the blocks."+
 		"Please note that this uses a NAIVE algorithm for merging (no smart replica deduplication, just chaining samples together)."+

cmd/thanos/downsample.go

@@ -73,7 +73,7 @@ func RunDownsample(
 		return err
 	}
 
-	metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
+	metaFetcher, err := block.NewMetaFetcher(logger, fetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
 		block.NewDeduplicateFilter(),
 	}, nil)
 	if err != nil {