From 88f0d2aa09c852cd724365828ffaa696a35bd287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 11 Nov 2020 17:55:32 +0100 Subject: [PATCH 1/3] Update Cortex to recent master (1.5.0+, 35e698bb56d6). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- go.mod | 5 +- go.sum | 16 +- pkg/distributor/distributor_test.go | 6 +- pkg/loki/loki.go | 6 +- pkg/loki/modules.go | 55 +- pkg/lokifrontend/config.go | 19 +- pkg/querier/ingester_querier.go | 6 +- pkg/querier/querier_mock_test.go | 6 +- pkg/querier/queryrange/roundtrip.go | 17 +- pkg/querier/queryrange/roundtrip_test.go | 15 +- pkg/util/validation/limits.go | 7 + .../cortex/pkg/alertmanager/alertmanager.go | 24 +- .../pkg/alertmanager/alertmanager_metrics.go | 14 +- .../cortex/pkg/alertmanager/api.go | 6 +- .../cortex/pkg/alertmanager/multitenant.go | 15 +- .../cortexproject/cortex/pkg/api/api.go | 193 +- .../cortexproject/cortex/pkg/api/handlers.go | 137 ++ .../pkg/chunk/cassandra/storage_client.go | 28 +- .../cortexproject/cortex/pkg/chunk/chunk.go | 43 +- .../cortex/pkg/chunk/chunk_store.go | 16 +- .../cortex/pkg/chunk/encoding/bigchunk.go | 4 + .../cortex/pkg/chunk/encoding/chunk.go | 54 +- .../cortex/pkg/chunk/encoding/doubledelta.go | 4 + .../cortex/pkg/chunk/encoding/varbit.go | 4 + .../cortex/pkg/chunk/grpc/grpc_client.go | 2 +- .../cortex/pkg/chunk/table_manager.go | 8 +- .../cortex/pkg/compactor/blocks_cleaner.go | 2 +- .../cortex/pkg/compactor/compactor.go | 71 +- .../cortexproject/cortex/pkg/cortex/cortex.go | 110 +- .../cortex/pkg/cortex/modules.go | 332 ++- .../cortex/pkg/distributor/distributor.go | 21 +- .../cortex/pkg/distributor/query.go | 8 +- .../cortex/pkg/frontend/config.go | 76 + .../pkg/frontend/downstream_roundtripper.go | 40 + .../cortex/pkg/frontend/transport/handler.go | 135 ++ .../pkg/frontend/transport/roundtripper.go | 47 + .../cortex/pkg/frontend/v1/frontend.go | 282 +++ .../v1/frontendv1pb}/frontend.pb.go | 57 +- .../v1/frontendv1pb}/frontend.proto | 4 +- .../cortex/pkg/frontend/v2/frontend.go | 309 +++ .../frontend/v2/frontend_scheduler_worker.go | 310 +++ .../frontend/v2/frontendv2pb/frontend.pb.go | 711 +++++++ .../frontend/v2/frontendv2pb/frontend.proto | 26 + .../cortex/pkg/ingester/client/cortex.pb.go | 351 +++- .../cortex/pkg/ingester/client/cortex.proto | 4 + .../cortex/pkg/ingester/ingester_v2.go | 313 ++- .../cortex/pkg/ingester/metrics.go | 60 +- .../cortexproject/cortex/pkg/ingester/wal.go | 7 +- .../cortex/pkg/querier/blocks_scanner.go | 2 +- .../pkg/querier/blocks_store_queryable.go | 14 +- .../pkg/querier/distributor_queryable.go | 8 +- .../cortex/pkg/querier/frontend/frontend.go | 531 ----- .../cortex/pkg/querier/frontend/worker.go | 219 -- .../frontend/worker_frontend_manager.go | 172 -- .../cortex/pkg/querier/querier.go | 109 +- .../cortex/pkg/querier/queryrange/limits.go | 111 +- .../pkg/querier/queryrange/query_range.go | 21 +- .../cortex/pkg/querier/queryrange/retry.go | 4 +- .../pkg/querier/queryrange/roundtrip.go | 22 +- .../cortex/pkg/querier/queryrange/util.go | 70 + .../pkg/querier/worker/frontend_processor.go | 127 ++ .../pkg/querier/worker/processor_manager.go | 75 + .../pkg/querier/worker/scheduler_processor.go | 210 ++ .../cortex/pkg/querier/worker/worker.go | 263 +++ .../pkg/ring/client/ring_service_discovery.go | 2 +- .../cortexproject/cortex/pkg/ring/model.go | 3 + .../cortexproject/cortex/pkg/ring/ring.go | 51 +- .../cortexproject/cortex/pkg/ruler/api.go | 8 +- .../cortexproject/cortex/pkg/ruler/ruler.go | 6 +- .../cortex/pkg/scheduler/queue/queue.go | 189 ++ .../queue/user_queues.go} | 10 +- .../cortex/pkg/scheduler/scheduler.go | 455 +++++ .../pkg/scheduler/schedulerpb/scheduler.pb.go | 1809 +++++++++++++++++ .../pkg/scheduler/schedulerpb/scheduler.proto | 85 + .../storage/backend/swift/bucket_client.go | 37 + .../pkg/storage/backend/swift/config.go | 46 + .../cortex/pkg/storage/tsdb/bucket_client.go | 3 + .../cortex/pkg/storage/tsdb/config.go | 21 +- .../storage/tsdb/memcache_client_config.go | 2 +- .../cortex/pkg/storegateway/bucket_stores.go | 2 +- .../cortex/pkg/storegateway/gateway.go | 4 +- .../cortex/pkg/util/dns_watcher.go | 82 + .../cortexproject/cortex/pkg/util/errors.go | 4 +- .../cortex/pkg/util/fakeauth/fake_auth.go | 22 +- .../cortex/pkg/util/grpcclient/grpcclient.go | 7 + .../cortex/pkg/util/grpcutil/carrier.go | 40 + .../cortex/pkg/util/metrics_helper.go | 7 + .../cortex/pkg/util/spanlogger/spanlogger.go | 2 +- .../cortex/pkg/util/test/poll.go | 2 +- .../cortexproject/cortex/pkg/util/time.go | 11 + .../cortex/pkg/util/validation/limits.go | 24 +- .../prometheus/discovery/discovery.go | 2 +- .../discovery/dockerswarm/network.go | 1 + .../discovery/openstack/instance.go | 1 + .../prometheus/discovery/registry.go | 3 +- .../prometheus/notifier/notifier.go | 4 +- .../prometheus/pkg/labels/labels.go | 36 +- .../prometheus/pkg/textparse/interface.go | 16 +- .../promql/parser/generated_parser.y.go | 7 +- .../prometheus/promql/parser/printer.go | 1 + .../prometheus/prometheus/promql/test.go | 1 + .../prometheus/prometheus/promql/value.go | 3 +- .../prometheus/prometheus/rules/alerting.go | 6 +- .../prometheus/prometheus/scrape/manager.go | 2 +- .../prometheus/prometheus/scrape/scrape.go | 39 +- .../prometheus/prometheus/storage/fanout.go | 10 +- .../prometheus/prometheus/storage/merge.go | 15 +- .../prometheus/storage/remote/client.go | 2 +- .../prometheus/storage/remote/codec.go | 1 + .../prometheus/storage/remote/intern.go | 3 +- .../remote/{max_gauge.go => max_timestamp.go} | 12 +- .../storage/remote/queue_manager.go | 21 +- .../prometheus/storage/remote/read.go | 1 + .../prometheus/storage/remote/storage.go | 4 +- .../prometheus/storage/remote/write.go | 7 +- .../prometheus/template/template.go | 7 +- .../prometheus/prometheus/tsdb/CHANGELOG.md | 2 +- .../prometheus/prometheus/tsdb/README.md | 8 +- .../prometheus/prometheus/tsdb/block.go | 29 +- .../prometheus/tsdb/chunkenc/chunk.go | 7 +- .../prometheus/tsdb/chunks/chunks.go | 40 +- .../prometheus/tsdb/chunks/head_chunks.go | 75 +- .../prometheus/prometheus/tsdb/compact.go | 36 +- .../prometheus/prometheus/tsdb/db.go | 211 +- .../prometheus/tsdb/errors/errors.go | 77 +- .../prometheus/tsdb/fileutil/mmap.go | 2 +- .../prometheus/prometheus/tsdb/head.go | 57 +- .../prometheus/prometheus/tsdb/index/index.go | 9 +- .../prometheus/prometheus/tsdb/querier.go | 13 +- .../prometheus/tsdb/record/record.go | 1 + .../prometheus/prometheus/tsdb/repair.go | 17 +- .../prometheus/tsdb/tombstones/tombstones.go | 7 +- .../prometheus/tsdb/tsdbblockutil.go | 1 + .../prometheus/prometheus/tsdb/wal.go | 1 + .../prometheus/tsdb/wal/checkpoint.go | 7 +- .../prometheus/prometheus/tsdb/wal/wal.go | 25 +- .../prometheus/prometheus/tsdb/wal/watcher.go | 1 + .../prometheus/util/testutil/directory.go | 18 +- .../prometheus/util/testutil/testing.go | 129 -- .../prometheus/prometheus/web/api/v1/api.go | 1 + .../thanos-io/thanos/pkg/block/fetcher.go | 9 +- .../thanos-io/thanos/pkg/compact/compact.go | 17 +- .../pkg/compact/downsample/downsample.go | 4 +- .../downsample/streamed_block_writer.go | 6 +- .../thanos/pkg/discovery/dns/provider.go | 4 +- .../thanos/pkg/errutil/multierror.go.go | 51 + .../thanos/pkg/objstore/azure/helpers.go | 11 + .../thanos-io/thanos/pkg/objstore/s3/s3.go | 28 +- .../thanos-io/thanos/pkg/runutil/runutil.go | 7 +- .../thanos-io/thanos/pkg/store/bucket.go | 57 +- .../thanos-io/thanos/pkg/store/multitsdb.go | 6 +- .../pkg/store/storepb/testutil/series.go | 10 +- .../x/tools/internal/gocommand/invoke.go | 30 +- .../x/tools/internal/imports/fix.go | 34 +- .../x/tools/internal/imports/mod.go | 26 +- vendor/modules.txt | 23 +- 156 files changed, 7873 insertions(+), 2307 deletions(-) create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/config.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/downstream_roundtripper.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/transport/handler.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/transport/roundtripper.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontend.go rename vendor/github.com/cortexproject/cortex/pkg/{querier/frontend => frontend/v1/frontendv1pb}/frontend.pb.go (89%) rename vendor/github.com/cortexproject/cortex/pkg/{querier/frontend => frontend/v1/frontendv1pb}/frontend.proto (82%) create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend_scheduler_worker.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.pb.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.proto delete mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go delete mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker.go delete mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker_frontend_manager.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/util.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/worker/frontend_processor.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/worker/processor_manager.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/worker/scheduler_processor.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/querier/worker/worker.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/queue.go rename vendor/github.com/cortexproject/cortex/pkg/{querier/frontend/frontend_querier_queues.go => scheduler/queue/user_queues.go} (97%) create mode 100644 vendor/github.com/cortexproject/cortex/pkg/scheduler/scheduler.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.pb.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.proto create mode 100644 vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/bucket_client.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/config.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/util/dns_watcher.go create mode 100644 vendor/github.com/cortexproject/cortex/pkg/util/grpcutil/carrier.go rename vendor/github.com/prometheus/prometheus/storage/remote/{max_gauge.go => max_timestamp.go} (80%) create mode 100644 vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go.go diff --git a/go.mod b/go.mod index 39ccb2aee545..e36f59d9aa45 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/grafana/loki go 1.15 require ( + github.com/NYTimes/gziphandler v1.1.1 github.com/aws/aws-lambda-go v1.17.0 github.com/blang/semver v3.5.1+incompatible // indirect github.com/bmatcuk/doublestar v1.2.2 @@ -10,7 +11,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.1 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf - github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf + github.com/cortexproject/cortex v1.5.1-0.20201110164020-35e698bb56d6 github.com/davecgh/go-spew v1.1.1 github.com/docker/docker v17.12.0-ce-rc1.0.20201009160326-9c15e82f19b0+incompatible github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect @@ -47,7 +48,7 @@ require ( github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.14.0 - github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643 + github.com/prometheus/prometheus v1.8.2-0.20201029103703-63be30dceed9 github.com/segmentio/fasthash v1.0.2 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 diff --git a/go.sum b/go.sum index 27a6e4b1520a..ba85665d20e9 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,9 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cortexproject/cortex v0.6.1-0.20200228110116-92ab6cbe0995/go.mod h1:3Xa3DjJxtpXqxcMGdk850lcIRb81M0fyY1MQ6udY134= github.com/cortexproject/cortex v1.2.1-0.20200805064754-d8edc95e2c91/go.mod h1:PVPxNLrxKH+yc8asaJOxuz7TiRmMizFfnSMOnRzM6oM= github.com/cortexproject/cortex v1.3.1-0.20200923145333-8587ea61fe17/go.mod h1:dJ9gpW7dzQ7z09cKtNN9PfebumgyO4dtNdFQ6eQEed0= -github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf h1:TGmSZFMNUP0U3bFMPeaVo1TFWlFaBtxrwS5Rs0zmFCs= -github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf/go.mod h1:MBJnS5mzVcHqivBp2391HpflMeMiT+f8r4VNkJlsZFs= +github.com/cortexproject/cortex v1.4.1-0.20201030080541-83ad6df2abea/go.mod h1:kXo5F3jlF7Ky3+I31jt/bXTzOlQjl2X/vGDpy0RY1gU= +github.com/cortexproject/cortex v1.5.1-0.20201110164020-35e698bb56d6 h1:H4EbbnbdKpX+9ngB2nscbAH1qxvfI69VGRWSxvNBpdY= +github.com/cortexproject/cortex v1.5.1-0.20201110164020-35e698bb56d6/go.mod h1:zFBGVsvRBfVp6ARXZ7pmiLaGlbjda5ZnA4Y6qSJyrQg= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= @@ -1145,8 +1146,9 @@ github.com/prometheus/prometheus v1.8.2-0.20200805082714-e0cf219f0de2/go.mod h1: github.com/prometheus/prometheus v1.8.2-0.20200819132913-cb830b0a9c78/go.mod h1:zfAqy/MwhMFajB9E2n12/9gG2fvofIE9uKDtlZCDxqs= github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3 h1:ETwF5e2G5PykV5usbsWoh1JfSbo50R07aMDdhTkC5SA= github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3/go.mod h1:9VNWoDFHOMovlubld5uKKxfCDcPBj2GMOCjcUFXkYaM= -github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643 h1:BDAexvKlOVjE5A8MlqRxzwkEpPl1/v6ydU1/J7kJtZc= -github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643/go.mod h1:XYjkJiog7fyQu3puQNivZPI2pNq1C/775EIoHfDvuvY= +github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24/go.mod h1:MDRkz271loM/PrYN+wUNEaTMDGSP760MQzB0yEjdgSQ= +github.com/prometheus/prometheus v1.8.2-0.20201029103703-63be30dceed9 h1:T6pkPNGKXv21lLfgD/mnIABj9aOhmz8HphDmKllfKWs= +github.com/prometheus/prometheus v1.8.2-0.20201029103703-63be30dceed9/go.mod h1:MDRkz271loM/PrYN+wUNEaTMDGSP760MQzB0yEjdgSQ= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1249,6 +1251,8 @@ github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763 h1:c84P3YUu8bx github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763/go.mod h1:KyW0a93tsh7v4hXAwo2CVAIRYuZT1Kkf4e04gisQjAg= github.com/thanos-io/thanos v0.13.1-0.20201019130456-f41940581d9a h1:4rNkFHeY+EIR7UdiYn5fZE7Q35Y3Dmae8q1Qbb90tcY= github.com/thanos-io/thanos v0.13.1-0.20201019130456-f41940581d9a/go.mod h1:A3qUEEbsVkplJnxyDLwuIuvTDaJPByTH+hMdTl9ujAA= +github.com/thanos-io/thanos v0.13.1-0.20201030101306-47f9a225cc52 h1:z3hglXVwJ4HgU0OoDS+8+MvEipv/U83IQ+fMsDr00YQ= +github.com/thanos-io/thanos v0.13.1-0.20201030101306-47f9a225cc52/go.mod h1:OqqX4x21cg5N5MMHd/yGQAc/V3wg8a7Do4Jk8HfaFZQ= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1674,8 +1678,8 @@ golang.org/x/tools v0.0.0-20200822203824-307de81be3f4 h1:r0nbB2EeRbGpnVeqxlkgiBp golang.org/x/tools v0.0.0-20200822203824-307de81be3f4/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201008025239-9df69603baec h1:RY2OghEV/7X1MLaecgm1mwFd3sGvUddm5pGVSxQvX0c= -golang.org/x/tools v0.0.0-20201008025239-9df69603baec/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca h1:pvScuB+UnCGDas2naNKUOXruM08MjwVcEdaweeynIqQ= +golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 9597f7b525b9..687882aa3051 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -310,7 +310,11 @@ func (r mockRing) Get(key uint32, op ring.Operation, buf []ring.IngesterDesc) (r return result, nil } -func (r mockRing) GetAll(op ring.Operation) (ring.ReplicationSet, error) { +func (r mockRing) GetAllHealthy(op ring.Operation) (ring.ReplicationSet, error) { + return r.GetReplicationSetForOperation(op) +} + +func (r mockRing) GetReplicationSetForOperation(op ring.Operation) (ring.ReplicationSet, error) { return ring.ReplicationSet{ Ingesters: r.ingesters, MaxErrors: 1, diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 0e999440c556..58691443794c 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -7,6 +7,9 @@ import ( "fmt" "net/http" + frontend "github.com/cortexproject/cortex/pkg/frontend/v1" + "github.com/cortexproject/cortex/pkg/querier/worker" + "github.com/grafana/loki/pkg/storage/stores/shipper/compactor" "github.com/cortexproject/cortex/pkg/util/flagext" @@ -15,7 +18,6 @@ import ( "github.com/weaveworks/common/signals" "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/memberlist" cortex_ruler "github.com/cortexproject/cortex/pkg/ruler" @@ -60,7 +62,7 @@ type Config struct { SchemaConfig storage.SchemaConfig `yaml:"schema_config,omitempty"` LimitsConfig validation.Limits `yaml:"limits_config,omitempty"` TableManager chunk.TableManagerConfig `yaml:"table_manager,omitempty"` - Worker frontend.WorkerConfig `yaml:"frontend_worker,omitempty"` + Worker worker.Config `yaml:"frontend_worker,omitempty"` Frontend lokifrontend.Config `yaml:"frontend,omitempty"` Ruler ruler.Config `yaml:"ruler,omitempty"` QueryRange queryrange.Config `yaml:"query_range,omitempty"` diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 0de2559c146a..5ab798d7e90c 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -9,6 +9,11 @@ import ( "os" "time" + "github.com/NYTimes/gziphandler" + "github.com/cortexproject/cortex/pkg/frontend" + "github.com/cortexproject/cortex/pkg/frontend/transport" + "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" + "github.com/grafana/loki/pkg/ruler/manager" "github.com/grafana/loki/pkg/storage/stores/shipper/compactor" @@ -18,8 +23,7 @@ import ( cortex_storage "github.com/cortexproject/cortex/pkg/chunk/storage" chunk_util "github.com/cortexproject/cortex/pkg/chunk/util" "github.com/cortexproject/cortex/pkg/cortex" - cortex_querier "github.com/cortexproject/cortex/pkg/querier" - "github.com/cortexproject/cortex/pkg/querier/frontend" + cortex_querier_worker "github.com/cortexproject/cortex/pkg/querier/worker" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" "github.com/cortexproject/cortex/pkg/ring/kv/memberlist" @@ -155,11 +159,21 @@ func (t *Loki) initDistributor() (services.Service, error) { } func (t *Loki) initQuerier() (services.Service, error) { - level.Debug(util.Logger).Log("msg", "initializing querier worker", "config", fmt.Sprintf("%+v", t.cfg.Worker)) - worker, err := frontend.NewWorker(t.cfg.Worker, cortex_querier.Config{MaxConcurrent: t.cfg.Querier.MaxConcurrent}, httpgrpc_server.NewServer(t.server.HTTPServer.Handler), util.Logger) - if err != nil { - return nil, err + var ( + worker services.Service + err error + ) + + // NewQuerierWorker now expects Frontend (or Scheduler) address to be set. Loki only supports Frontend for now. + if t.cfg.Worker.FrontendAddress != "" { + t.cfg.Worker.MaxConcurrentRequests = t.cfg.Querier.MaxConcurrent + level.Debug(util.Logger).Log("msg", "initializing querier worker", "config", fmt.Sprintf("%+v", t.cfg.Worker)) + worker, err = cortex_querier_worker.NewQuerierWorker(t.cfg.Worker, httpgrpc_server.NewServer(t.server.HTTPServer.Handler), util.Logger, prometheus.DefaultRegisterer) + if err != nil { + return nil, err + } } + if t.cfg.Ingester.QueryStoreMaxLookBackPeriod != 0 { t.cfg.Querier.IngesterQueryStoreMaxLookback = t.cfg.Ingester.QueryStoreMaxLookBackPeriod } @@ -339,12 +353,23 @@ type disabledShuffleShardingLimits struct{} func (disabledShuffleShardingLimits) MaxQueriersPerUser(userID string) int { return 0 } func (t *Loki) initQueryFrontend() (_ services.Service, err error) { - level.Debug(util.Logger).Log("msg", "initializing query frontend", "config", fmt.Sprintf("%+v", t.cfg.Frontend)) - t.frontend, err = frontend.New(t.cfg.Frontend.Config, disabledShuffleShardingLimits{}, util.Logger, prometheus.DefaultRegisterer) + + roundTripper, frontendV1, _, err := frontend.InitFrontend(frontend.CombinedFrontendConfig{ + // Don't set FrontendV2 field to make sure that only frontendV1 can be initialized. + Handler: t.cfg.Frontend.Handler, + FrontendV1: t.cfg.Frontend.FrontendV1, + CompressResponses: t.cfg.Frontend.CompressResponses, + DownstreamURL: t.cfg.Frontend.DownstreamURL, + }, disabledShuffleShardingLimits{}, t.cfg.Server.GRPCListenPort, util.Logger, prometheus.DefaultRegisterer) if err != nil { - return + return nil, err } + t.frontend = frontendV1 + if t.frontend != nil { + frontendv1pb.RegisterFrontendServer(t.server.GRPC, t.frontend) + } + level.Debug(util.Logger).Log("msg", "initializing query range tripperware", "config", fmt.Sprintf("%+v", t.cfg.QueryRange), "limits", fmt.Sprintf("%+v", t.cfg.LimitsConfig), @@ -361,16 +386,20 @@ func (t *Loki) initQueryFrontend() (_ services.Service, err error) { return } t.stopper = stopper - t.frontend.Wrap(tripperware) - frontend.RegisterFrontendServer(t.server.GRPC, t.frontend) - frontendHandler := middleware.Merge( + roundTripper = tripperware(roundTripper) + frontendHandler := transport.NewHandler(t.cfg.Frontend.Handler, roundTripper, util.Logger) + if t.cfg.Frontend.CompressResponses { + frontendHandler = gziphandler.GzipHandler(frontendHandler) + } + + frontendHandler = middleware.Merge( serverutil.RecoveryHTTPMiddleware, t.httpAuthMiddleware, queryrange.StatsHTTPMiddleware, serverutil.NewPrepopulateMiddleware(), serverutil.ResponseJSONMiddleware(), - ).Wrap(t.frontend.Handler()) + ).Wrap(frontendHandler) var defaultHandler http.Handler if t.cfg.Frontend.TailProxyURL != "" { diff --git a/pkg/lokifrontend/config.go b/pkg/lokifrontend/config.go index 800c517e4c7f..ba0826c3d4f3 100644 --- a/pkg/lokifrontend/config.go +++ b/pkg/lokifrontend/config.go @@ -3,16 +3,27 @@ package lokifrontend import ( "flag" - "github.com/cortexproject/cortex/pkg/querier/frontend" + "github.com/cortexproject/cortex/pkg/frontend/transport" + "github.com/cortexproject/cortex/pkg/frontend/v1" ) type Config struct { - frontend.Config `yaml:",inline"` - TailProxyURL string `yaml:"tail_proxy_url"` + Handler transport.HandlerConfig `yaml:",inline"` + FrontendV1 v1.Config `yaml:",inline"` + + CompressResponses bool `yaml:"compress_responses"` + DownstreamURL string `yaml:"downstream_url"` + + TailProxyURL string `yaml:"tail_proxy_url"` } // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.Config.RegisterFlags(f) + cfg.Handler.RegisterFlags(f) + cfg.FrontendV1.RegisterFlags(f) + + f.BoolVar(&cfg.CompressResponses, "querier.compress-http-responses", false, "Compress HTTP responses.") + f.StringVar(&cfg.DownstreamURL, "frontend.downstream-url", "", "URL of downstream Prometheus.") + f.StringVar(&cfg.TailProxyURL, "frontend.tail-proxy-url", "", "URL of querier for tail proxy.") } diff --git a/pkg/querier/ingester_querier.go b/pkg/querier/ingester_querier.go index 2ebf8a29c0c0..88735c36955b 100644 --- a/pkg/querier/ingester_querier.go +++ b/pkg/querier/ingester_querier.go @@ -63,7 +63,7 @@ func newIngesterQuerier(clientCfg client.Config, ring ring.ReadRing, extraQueryD // forAllIngesters runs f, in parallel, for all ingesters // TODO taken from Cortex, see if we can refactor out an usable interface. func (q *IngesterQuerier) forAllIngesters(ctx context.Context, f func(logproto.QuerierClient) (interface{}, error)) ([]responseFromIngesters, error) { - replicationSet, err := q.ring.GetAll(ring.Read) + replicationSet, err := q.ring.GetReplicationSetForOperation(ring.Read) if err != nil { return nil, err } @@ -169,7 +169,7 @@ func (q *IngesterQuerier) TailDisconnectedIngesters(ctx context.Context, req *lo } // Get the current replication set from the ring - replicationSet, err := q.ring.GetAll(ring.Read) + replicationSet, err := q.ring.GetReplicationSetForOperation(ring.Read) if err != nil { return nil, err } @@ -226,7 +226,7 @@ func (q *IngesterQuerier) Series(ctx context.Context, req *logproto.SeriesReques } func (q *IngesterQuerier) TailersCount(ctx context.Context) ([]uint32, error) { - replicationSet, err := q.ring.GetAll(ring.Read) + replicationSet, err := q.ring.GetAllHealthy(ring.Read) if err != nil { return nil, err } diff --git a/pkg/querier/querier_mock_test.go b/pkg/querier/querier_mock_test.go index 4439743f70a8..e5f5e54255e3 100644 --- a/pkg/querier/querier_mock_test.go +++ b/pkg/querier/querier_mock_test.go @@ -319,7 +319,11 @@ func (r *readRingMock) BatchGet(keys []uint32, op ring.Operation) ([]ring.Replic return []ring.ReplicationSet{r.replicationSet}, nil } -func (r *readRingMock) GetAll(op ring.Operation) (ring.ReplicationSet, error) { +func (r *readRingMock) GetAllHealthy(op ring.Operation) (ring.ReplicationSet, error) { + return r.replicationSet, nil +} + +func (r *readRingMock) GetReplicationSetForOperation(op ring.Operation) (ring.ReplicationSet, error) { return r.replicationSet, nil } diff --git a/pkg/querier/queryrange/roundtrip.go b/pkg/querier/queryrange/roundtrip.go index ae2a850f2505..e4c33945801e 100644 --- a/pkg/querier/queryrange/roundtrip.go +++ b/pkg/querier/queryrange/roundtrip.go @@ -9,7 +9,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/querier/queryrange" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -45,7 +44,7 @@ func NewTripperware( schema chunk.SchemaConfig, minShardingLookback time.Duration, registerer prometheus.Registerer, -) (frontend.Tripperware, Stopper, error) { +) (queryrange.Tripperware, Stopper, error) { // Ensure that QuerySplitDuration uses configuration defaults. // This avoids divide by zero errors when determining cache keys where user specific overrides don't exist. limits = WithDefaultLimits(limits, cfg.Config) @@ -222,8 +221,8 @@ func NewLogFilterTripperware( retryMiddlewareMetrics *queryrange.RetryMiddlewareMetrics, shardingMetrics *logql.ShardingMetrics, splitByMetrics *SplitByMetrics, -) (frontend.Tripperware, error) { - queryRangeMiddleware := []queryrange.Middleware{StatsCollectorMiddleware(), queryrange.LimitsMiddleware(limits)} +) (queryrange.Tripperware, error) { + queryRangeMiddleware := []queryrange.Middleware{StatsCollectorMiddleware(), queryrange.NewLimitsMiddleware(limits)} if cfg.SplitQueriesByInterval != 0 { queryRangeMiddleware = append(queryRangeMiddleware, queryrange.InstrumentMiddleware("split_by_interval", instrumentMetrics), SplitByIntervalMiddleware(limits, codec, splitByMetrics)) } @@ -264,7 +263,7 @@ func NewSeriesTripperware( instrumentMetrics *queryrange.InstrumentMiddlewareMetrics, retryMiddlewareMetrics *queryrange.RetryMiddlewareMetrics, splitByMetrics *SplitByMetrics, -) (frontend.Tripperware, error) { +) (queryrange.Tripperware, error) { queryRangeMiddleware := []queryrange.Middleware{} if cfg.SplitQueriesByInterval != 0 { queryRangeMiddleware = append(queryRangeMiddleware, @@ -294,7 +293,7 @@ func NewLabelsTripperware( instrumentMetrics *queryrange.InstrumentMiddlewareMetrics, retryMiddlewareMetrics *queryrange.RetryMiddlewareMetrics, splitByMetrics *SplitByMetrics, -) (frontend.Tripperware, error) { +) (queryrange.Tripperware, error) { queryRangeMiddleware := []queryrange.Middleware{} if cfg.SplitQueriesByInterval != 0 { queryRangeMiddleware = append(queryRangeMiddleware, @@ -330,8 +329,8 @@ func NewMetricTripperware( shardingMetrics *logql.ShardingMetrics, splitByMetrics *SplitByMetrics, registerer prometheus.Registerer, -) (frontend.Tripperware, Stopper, error) { - queryRangeMiddleware := []queryrange.Middleware{StatsCollectorMiddleware(), queryrange.LimitsMiddleware(limits)} +) (queryrange.Tripperware, Stopper, error) { + queryRangeMiddleware := []queryrange.Middleware{StatsCollectorMiddleware(), queryrange.NewLimitsMiddleware(limits)} if cfg.AlignQueriesWithStep { queryRangeMiddleware = append( queryRangeMiddleware, @@ -404,7 +403,7 @@ func NewMetricTripperware( // Finally, if the user selected any query range middleware, stitch it in. if len(queryRangeMiddleware) > 0 { rt := queryrange.NewRoundTripper(next, codec, queryRangeMiddleware...) - return frontend.RoundTripFunc(func(r *http.Request) (*http.Response, error) { + return queryrange.RoundTripFunc(func(r *http.Request) (*http.Response, error) { if !strings.HasSuffix(r.URL.Path, "/query_range") { return next.RoundTrip(r) } diff --git a/pkg/querier/queryrange/roundtrip_test.go b/pkg/querier/queryrange/roundtrip_test.go index e05e524dde8e..096e0890a068 100644 --- a/pkg/querier/queryrange/roundtrip_test.go +++ b/pkg/querier/queryrange/roundtrip_test.go @@ -14,7 +14,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/querier/queryrange" "github.com/cortexproject/cortex/pkg/util" "github.com/prometheus/prometheus/pkg/labels" @@ -404,22 +403,22 @@ func TestPostQueries(t *testing.T) { req = req.WithContext(user.InjectOrgID(context.Background(), "1")) require.NoError(t, err) _, err = newRoundTripper( - frontend.RoundTripFunc(func(*http.Request) (*http.Response, error) { + queryrange.RoundTripFunc(func(*http.Request) (*http.Response, error) { t.Error("unexpected default roundtripper called") return nil, nil }), - frontend.RoundTripFunc(func(*http.Request) (*http.Response, error) { + queryrange.RoundTripFunc(func(*http.Request) (*http.Response, error) { return nil, nil }), - frontend.RoundTripFunc(func(*http.Request) (*http.Response, error) { + queryrange.RoundTripFunc(func(*http.Request) (*http.Response, error) { t.Error("unexpected metric roundtripper called") return nil, nil }), - frontend.RoundTripFunc(func(*http.Request) (*http.Response, error) { + queryrange.RoundTripFunc(func(*http.Request) (*http.Response, error) { t.Error("unexpected series roundtripper called") return nil, nil }), - frontend.RoundTripFunc(func(*http.Request) (*http.Response, error) { + queryrange.RoundTripFunc(func(*http.Request) (*http.Response, error) { t.Error("unexpected labels roundtripper called") return nil, nil }), @@ -522,6 +521,10 @@ func (f fakeLimits) MaxCacheFreshness(string) time.Duration { return 1 * time.Minute } +func (f fakeLimits) MaxQueryLookback(string) time.Duration { + return 0 +} + func counter() (*int, http.Handler) { count := 0 var lock sync.Mutex diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 28a4676f3d33..bf1016e41f0d 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -39,6 +39,7 @@ type Limits struct { // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_chunks_per_query"` + MaxQueryLookback time.Duration `yaml:"max_query_lookback"` MaxQueryLength time.Duration `yaml:"max_query_length"` MaxQueryParallelism int `yaml:"max_query_parallelism"` CardinalityLimit int `yaml:"cardinality_limit"` @@ -75,6 +76,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxChunksPerQuery, "store.query-chunk-limit", 2e6, "Maximum number of chunks that can be fetched in a single query.") f.DurationVar(&l.MaxQueryLength, "store.max-query-length", 0, "Limit to length of chunk store queries, 0 to disable.") + f.DurationVar(&l.MaxQueryLookback, "querier.max-query-lookback", 0, "Limit how long back data (series and metadata) can be queried, up until duration ago. This limit is enforced in the query-frontend, querier and ruler. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of queries will be scheduled in parallel by the frontend.") f.IntVar(&l.CardinalityLimit, "store.cardinality-limit", 1e5, "Cardinality limit for index queries.") f.IntVar(&l.MaxStreamsMatchersPerQuery, "querier.max-streams-matcher-per-query", 1000, "Limit the number of streams matchers per query") @@ -249,6 +251,11 @@ func (o *Overrides) MaxCacheFreshness(userID string) time.Duration { return o.getOverridesForUser(userID).MaxCacheFreshness } +// MaxQueryLookback returns the max lookback period of queries. +func (o *Overrides) MaxQueryLookback(userID string) time.Duration { + return o.getOverridesForUser(userID).MaxQueryLookback +} + func (o *Overrides) getOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { l := o.tenantLimits(userID) diff --git a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go index 98b507b0348e..ceed4e7ba199 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go +++ b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go @@ -2,6 +2,8 @@ package alertmanager import ( "context" + "crypto/md5" + "encoding/binary" "fmt" "net/http" "net/url" @@ -32,6 +34,7 @@ import ( "github.com/prometheus/alertmanager/types" "github.com/prometheus/alertmanager/ui" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/common/route" ) @@ -70,6 +73,11 @@ type Alertmanager struct { // The Dispatcher is the only component we need to recreate when we call ApplyConfig. // Given its metrics don't have any variable labels we need to re-use the same metrics. dispatcherMetrics *dispatch.DispatcherMetrics + // This needs to be set to the hash of the config. All the hashes need to be same + // for deduping of alerts to work, hence we need this metric. See https://github.com/prometheus/alertmanager/issues/596 + // Further, in upstream AM, this metric is handled using the config coordinator which we don't use + // hence we need to generate the metric ourselves. + configHashMetric prometheus.Gauge activeMtx sync.Mutex active bool @@ -97,6 +105,10 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { stop: make(chan struct{}), active: false, activeMtx: sync.Mutex{}, + configHashMetric: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "alertmanager_config_hash", + Help: "Hash of the currently loaded alertmanager configuration.", + }), } am.registry = reg @@ -182,7 +194,7 @@ func clusterWait(p *cluster.Peer, timeout time.Duration) func() time.Duration { } // ApplyConfig applies a new configuration to an Alertmanager. -func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config) error { +func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config, rawCfg string) error { templateFiles := make([]string, len(conf.Templates)) if len(conf.Templates) > 0 { for i, t := range conf.Templates { @@ -249,6 +261,7 @@ func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config) error { am.active = true am.activeMtx.Unlock() + am.configHashMetric.Set(md5HashAsMetricValue([]byte(rawCfg))) return nil } @@ -367,3 +380,12 @@ func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, log } return integrations, nil } + +func md5HashAsMetricValue(data []byte) float64 { + sum := md5.Sum(data) + // We only want 48 bits as a float64 only has a 53 bit mantissa. + smallSum := sum[0:6] + var bytes = make([]byte, 8) + copy(bytes, smallSum) + return float64(binary.LittleEndian.Uint64(bytes)) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager_metrics.go b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager_metrics.go index 354370d016a8..62617bb70366 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager_metrics.go +++ b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager_metrics.go @@ -45,6 +45,9 @@ type alertmanagerMetrics struct { silencesQueryDuration *prometheus.Desc silences *prometheus.Desc silencesPropagatedMessagesTotal *prometheus.Desc + + // The alertmanager config hash. + configHashValue *prometheus.Desc } func newAlertmanagerMetrics() *alertmanagerMetrics { @@ -135,6 +138,10 @@ func newAlertmanagerMetrics() *alertmanagerMetrics { "cortex_alertmanager_silences", "How many silences by state.", []string{"user", "state"}, nil), + configHashValue: prometheus.NewDesc( + "cortex_alertmanager_config_hash", + "Hash of the currently loaded alertmanager configuration.", + []string{"user"}, nil), } } @@ -162,6 +169,7 @@ func (m *alertmanagerMetrics) Describe(out chan<- *prometheus.Desc) { out <- m.numNotifications out <- m.numFailedNotifications out <- m.notificationLatencySeconds + out <- m.markerAlerts out <- m.nflogGCDuration out <- m.nflogSnapshotDuration out <- m.nflogSnapshotSize @@ -169,15 +177,15 @@ func (m *alertmanagerMetrics) Describe(out chan<- *prometheus.Desc) { out <- m.nflogQueryErrorsTotal out <- m.nflogQueryDuration out <- m.nflogPropagatedMessagesTotal - out <- m.markerAlerts out <- m.silencesGCDuration out <- m.silencesSnapshotDuration out <- m.silencesSnapshotSize out <- m.silencesQueriesTotal out <- m.silencesQueryErrorsTotal out <- m.silencesQueryDuration - out <- m.silences out <- m.silencesPropagatedMessagesTotal + out <- m.silences + out <- m.configHashValue } func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { @@ -207,4 +215,6 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfHistograms(out, m.silencesQueryDuration, "alertmanager_silences_query_duration_seconds") data.SendSumOfCounters(out, m.silencesPropagatedMessagesTotal, "alertmanager_silences_gossip_messages_propagated_total") data.SendSumOfGaugesPerUserWithLabels(out, m.silences, "alertmanager_silences", "state") + + data.SendMaxOfGaugesPerUser(out, m.configHashValue, "alertmanager_config_hash") } diff --git a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/api.go b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/api.go index 68e75d7556b3..2a32b5f8120a 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/api.go +++ b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/api.go @@ -36,7 +36,7 @@ type UserConfig struct { func (am *MultitenantAlertmanager) GetUserConfig(w http.ResponseWriter, r *http.Request) { logger := util.WithContext(r.Context(), am.logger) - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + userID, err := user.ExtractOrgID(r.Context()) if err != nil { level.Error(logger).Log("msg", errNoOrgID, "err", err.Error()) http.Error(w, fmt.Sprintf("%s: %s", errNoOrgID, err.Error()), http.StatusUnauthorized) @@ -73,7 +73,7 @@ func (am *MultitenantAlertmanager) GetUserConfig(w http.ResponseWriter, r *http. func (am *MultitenantAlertmanager) SetUserConfig(w http.ResponseWriter, r *http.Request) { logger := util.WithContext(r.Context(), am.logger) - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + userID, err := user.ExtractOrgID(r.Context()) if err != nil { level.Error(logger).Log("msg", errNoOrgID, "err", err.Error()) http.Error(w, fmt.Sprintf("%s: %s", errNoOrgID, err.Error()), http.StatusUnauthorized) @@ -114,7 +114,7 @@ func (am *MultitenantAlertmanager) SetUserConfig(w http.ResponseWriter, r *http. func (am *MultitenantAlertmanager) DeleteUserConfig(w http.ResponseWriter, r *http.Request) { logger := util.WithContext(r.Context(), am.logger) - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + userID, err := user.ExtractOrgID(r.Context()) if err != nil { level.Error(logger).Log("msg", errNoOrgID, "err", err.Error()) http.Error(w, fmt.Sprintf("%s: %s", errNoOrgID, err.Error()), http.StatusUnauthorized) diff --git a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go index 5db09e63eeb1..1ea6a4ce5c4e 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go +++ b/vendor/github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go @@ -391,6 +391,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alerts.AlertConfigDesc) error { level.Debug(am.logger).Log("msg", "setting config", "user", cfg.User) + rawCfg := cfg.RawConfig if cfg.RawConfig == "" { if am.fallbackConfig == "" { return fmt.Errorf("blank Alertmanager configuration for %v", cfg.User) @@ -400,6 +401,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alerts.AlertConfigDesc) error { if err != nil { return fmt.Errorf("unable to load fallback configuration for %v: %v", cfg.User, err) } + rawCfg = am.fallbackConfig } else { userAmConfig, err = amconfig.Load(cfg.RawConfig) if err != nil && hasExisting { @@ -419,7 +421,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alerts.AlertConfigDesc) error { // If no Alertmanager instance exists for this user yet, start one. if !hasExisting { level.Debug(am.logger).Log("msg", "initializing new per-tenant alertmanager", "user", cfg.User) - newAM, err := am.newAlertmanager(cfg.User, userAmConfig) + newAM, err := am.newAlertmanager(cfg.User, userAmConfig, rawCfg) if err != nil { return err } @@ -429,7 +431,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alerts.AlertConfigDesc) error { } else if am.cfgs[cfg.User].RawConfig != cfg.RawConfig || hasTemplateChanges { level.Info(am.logger).Log("msg", "updating new per-tenant alertmanager", "user", cfg.User) // If the config changed, apply the new one. - err := existing.ApplyConfig(cfg.User, userAmConfig) + err := existing.ApplyConfig(cfg.User, userAmConfig, rawCfg) if err != nil { return fmt.Errorf("unable to apply Alertmanager config for user %v: %v", cfg.User, err) } @@ -438,7 +440,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alerts.AlertConfigDesc) error { return nil } -func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amconfig.Config) (*Alertmanager, error) { +func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amconfig.Config, rawCfg string) (*Alertmanager, error) { reg := prometheus.NewRegistry() newAM, err := New(&Config{ UserID: userID, @@ -453,7 +455,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err) } - if err := newAM.ApplyConfig(userID, amConfig); err != nil { + if err := newAM.ApplyConfig(userID, amConfig, rawCfg); err != nil { return nil, fmt.Errorf("unable to apply initial config for user %v: %v", userID, err) } @@ -463,7 +465,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco // ServeHTTP serves the Alertmanager's web UI and API. func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) + userID, err := user.ExtractOrgID(req.Context()) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return @@ -474,6 +476,7 @@ func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Re if ok { if !userAM.IsActive() { + level.Debug(am.logger).Log("msg", "the Alertmanager is not active", "user", userID) http.Error(w, "the Alertmanager is not configured", http.StatusNotFound) return } @@ -485,6 +488,7 @@ func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Re if am.fallbackConfig != "" { userAM, err = am.alertmanagerFromFallbackConfig(userID) if err != nil { + level.Error(am.logger).Log("msg", "unable to initialize the Alertmanager with a fallback configuration", "user", userID, "err", err) http.Error(w, "Failed to initialize the Alertmanager", http.StatusInternalServerError) return } @@ -493,6 +497,7 @@ func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Re return } + level.Debug(am.logger).Log("msg", "the Alertmanager has no configuration and no fallback specified", "user", userID) http.Error(w, "the Alertmanager is not configured", http.StatusNotFound) } diff --git a/vendor/github.com/cortexproject/cortex/pkg/api/api.go b/vendor/github.com/cortexproject/cortex/pkg/api/api.go index 5b04d66022b2..5c32140028b9 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/api/api.go +++ b/vendor/github.com/cortexproject/cortex/pkg/api/api.go @@ -2,27 +2,16 @@ package api import ( "context" - "errors" "flag" "net/http" - "regexp" "strings" "time" - "github.com/opentracing-contrib/go-stdlib/nethttp" - "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/felixge/fgprof" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/gorilla/mux" - "github.com/prometheus/common/route" - "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/promql" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/storage" - v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" @@ -30,11 +19,16 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/compactor" "github.com/cortexproject/cortex/pkg/distributor" + frontendv1 "github.com/cortexproject/cortex/pkg/frontend/v1" + "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" + frontendv2 "github.com/cortexproject/cortex/pkg/frontend/v2" + "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/querier" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/scheduler" + "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/push" @@ -62,12 +56,13 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { } type API struct { - cfg Config - authMiddleware middleware.Interface - server *server.Server - logger log.Logger - sourceIPs *middleware.SourceIPExtractor - indexPage *IndexPageContent + AuthMiddleware middleware.Interface + + cfg Config + server *server.Server + logger log.Logger + sourceIPs *middleware.SourceIPExtractor + indexPage *IndexPageContent } func New(cfg Config, serverCfg server.Config, s *server.Server, logger log.Logger) (*API, error) { @@ -86,7 +81,7 @@ func New(cfg Config, serverCfg server.Config, s *server.Server, logger log.Logge api := &API{ cfg: cfg, - authMiddleware: cfg.HTTPAuthMiddleware, + AuthMiddleware: cfg.HTTPAuthMiddleware, server: s, logger: logger, sourceIPs: sourceIPs, @@ -95,7 +90,7 @@ func New(cfg Config, serverCfg server.Config, s *server.Server, logger log.Logge // If no authentication middleware is present in the config, use the default authentication middleware. if cfg.HTTPAuthMiddleware == nil { - api.authMiddleware = middleware.AuthenticateUser + api.AuthMiddleware = middleware.AuthenticateUser } return api, nil @@ -104,29 +99,23 @@ func New(cfg Config, serverCfg server.Config, s *server.Server, logger log.Logge // RegisterRoute registers a single route enforcing HTTP methods. A single // route is expected to be specific about which HTTP methods are supported. func (a *API) RegisterRoute(path string, handler http.Handler, auth bool, method string, methods ...string) { - a.registerRouteWithRouter(a.server.HTTP, path, handler, auth, method, methods...) -} - -// RegisterRoute registers a single route to a router, enforcing HTTP methods. A single -// route is expected to be specific about which HTTP methods are supported. -func (a *API) registerRouteWithRouter(router *mux.Router, path string, handler http.Handler, auth bool, method string, methods ...string) { methods = append([]string{method}, methods...) level.Debug(a.logger).Log("msg", "api: registering route", "methods", strings.Join(methods, ","), "path", path, "auth", auth) if auth { - handler = a.authMiddleware.Wrap(handler) + handler = a.AuthMiddleware.Wrap(handler) } if len(methods) == 0 { - router.Path(path).Handler(handler) + a.server.HTTP.Path(path).Handler(handler) return } - router.Path(path).Methods(methods...).Handler(handler) + a.server.HTTP.Path(path).Methods(methods...).Handler(handler) } func (a *API) RegisterRoutesWithPrefix(prefix string, handler http.Handler, auth bool, methods ...string) { level.Debug(a.logger).Log("msg", "api: registering route", "methods", strings.Join(methods, ","), "prefix", prefix, "auth", auth) if auth { - handler = a.authMiddleware.Wrap(handler) + handler = a.AuthMiddleware.Wrap(handler) } if len(methods) == 0 { a.server.HTTP.PathPrefix(prefix).Handler(handler) @@ -135,20 +124,6 @@ func (a *API) RegisterRoutesWithPrefix(prefix string, handler http.Handler, auth a.server.HTTP.PathPrefix(prefix).Methods(methods...).Handler(handler) } -// Latest Prometheus requires r.RemoteAddr to be set to addr:port, otherwise it reject the request. -// Requests to Querier sometimes doesn't have that (if they are fetched from Query-Frontend). -// Prometheus uses this when logging queries to QueryLogger, but Cortex doesn't call engine.SetQueryLogger to set one. -// -// Can be removed when (if) https://github.com/prometheus/prometheus/pull/6840 is merged. -func fakeRemoteAddr(handler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RemoteAddr == "" { - r.RemoteAddr = "127.0.0.1:8888" - } - handler.ServeHTTP(w, r) - }) -} - // RegisterAlertmanager registers endpoints associated with the alertmanager. It will only // serve endpoints using the legacy http-prefix if it is not run as a single binary. func (a *API) RegisterAlertmanager(am *alertmanager.MultitenantAlertmanager, target, apiEnabled bool) { @@ -200,9 +175,9 @@ func (a *API) RegisterDistributor(d *distributor.Distributor, pushConfig distrib a.RegisterRoute("/ha-tracker", d.HATracker, false, "GET") } -// ingester is defined as an interface to allow for alternative implementations +// Ingester is defined as an interface to allow for alternative implementations // of ingesters to be passed into the API.RegisterIngester() method. -type ingester interface { +type Ingester interface { client.IngesterServer FlushHandler(http.ResponseWriter, *http.Request) ShutdownHandler(http.ResponseWriter, *http.Request) @@ -210,7 +185,7 @@ type ingester interface { } // RegisterIngester registers the ingesters HTTP and GRPC service -func (a *API) RegisterIngester(i ingester, pushConfig distributor.Config) { +func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) { client.RegisterIngesterServer(a.server.GRPC, i) a.indexPage.AddLink(SectionDangerous, "/ingester/flush", "Trigger a Flush of data from Ingester to storage") @@ -302,114 +277,22 @@ func (a *API) RegisterCompactor(c *compactor.Compactor) { a.RegisterRoute("/compactor/ring", http.HandlerFunc(c.RingHandler), false, "GET", "POST") } -// RegisterQuerier registers the Prometheus routes supported by the -// Cortex querier service. Currently this can not be registered simultaneously -// with the QueryFrontend. -func (a *API) RegisterQuerier( +// RegisterQueryable registers the the default routes associated with the querier +// module. +func (a *API) RegisterQueryable( queryable storage.SampleAndChunkQueryable, - engine *promql.Engine, distributor *distributor.Distributor, - registerRoutesExternally bool, - tombstonesLoader *purger.TombstonesLoader, - querierRequestDuration *prometheus.HistogramVec, - receivedMessageSize *prometheus.HistogramVec, - sentMessageSize *prometheus.HistogramVec, - inflightRequests *prometheus.GaugeVec, -) http.Handler { - api := v1.NewAPI( - engine, - errorTranslateQueryable{queryable}, // Translate errors to errors expected by API. - func(context.Context) v1.TargetRetriever { return &querier.DummyTargetRetriever{} }, - func(context.Context) v1.AlertmanagerRetriever { return &querier.DummyAlertmanagerRetriever{} }, - func() config.Config { return config.Config{} }, - map[string]string{}, // TODO: include configuration flags - v1.GlobalURLOptions{}, - func(f http.HandlerFunc) http.HandlerFunc { return f }, - nil, // Only needed for admin APIs. - "", // This is for snapshots, which is disabled when admin APIs are disabled. Hence empty. - false, // Disable admin APIs. - a.logger, - func(context.Context) v1.RulesRetriever { return &querier.DummyRulesRetriever{} }, - 0, 0, 0, // Remote read samples and concurrency limit. - regexp.MustCompile(".*"), - func() (v1.RuntimeInfo, error) { return v1.RuntimeInfo{}, errors.New("not implemented") }, - &v1.PrometheusVersion{}, - // This is used for the stats API which we should not support. Or find other ways to. - prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return nil, nil }), - ) - +) { // these routes are always registered to the default server a.RegisterRoute("/api/v1/user_stats", http.HandlerFunc(distributor.UserStatsHandler), true, "GET") a.RegisterRoute("/api/v1/chunks", querier.ChunksHandler(queryable), true, "GET") a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/user_stats", http.HandlerFunc(distributor.UserStatsHandler), true, "GET") a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/chunks", querier.ChunksHandler(queryable), true, "GET") - - // these routes are either registered the default server OR to an internal mux. The internal mux is - // for use in a single binary mode when both the query frontend and the querier would attempt to claim these routes - // TODO: Add support to expose querier paths with a configurable prefix in single binary mode. - router := mux.NewRouter() - if registerRoutesExternally { - router = a.server.HTTP - } - - // Use a separate metric for the querier in order to differentiate requests from the query-frontend when - // running Cortex as a single binary. - inst := middleware.Instrument{ - RouteMatcher: router, - Duration: querierRequestDuration, - RequestBodySize: receivedMessageSize, - ResponseBodySize: sentMessageSize, - InflightRequests: inflightRequests, - } - - promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1") - api.Register(promRouter) - cacheGenHeaderMiddleware := getHTTPCacheGenNumberHeaderSetterMiddleware(tombstonesLoader) - promHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(promRouter))) - - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", promHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", promHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", promHandler, true, "GET") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/series", promHandler, true, "GET", "POST", "DELETE") - //TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: - // https://github.com/prometheus/prometheus/pull/7125/files - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", querier.MetadataHandler(distributor), true, "GET") - - legacyPromRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.LegacyHTTPPrefix + "/api/v1") - api.Register(legacyPromRouter) - legacyPromHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(legacyPromRouter))) - - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", legacyPromHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/labels", legacyPromHandler, true, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", legacyPromHandler, true, "GET") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/series", legacyPromHandler, true, "GET", "POST", "DELETE") - //TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: - // https://github.com/prometheus/prometheus/pull/7125/files - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", querier.MetadataHandler(distributor), true, "GET") - - // if we have externally registered routes then we need to return the server handler - // so that we continue to use all standard middleware - if registerRoutesExternally { - return a.server.HTTPServer.Handler - } - - // Since we have a new router and the request will not go trough the default server - // HTTP middleware stack, we need to add a middleware to extract the trace context - // from the HTTP headers and inject it into the Go context. - return nethttp.MiddlewareFunc(opentracing.GlobalTracer(), router.ServeHTTP, nethttp.OperationNameFunc(func(r *http.Request) string { - return "internalQuerier" - })) } -// registerQueryAPI registers the Prometheus routes supported by the -// Cortex querier service. Currently this can not be registered simultaneously -// with the Querier. -func (a *API) registerQueryAPI(handler http.Handler) { +// RegisterQueryAPI registers the Prometheus API routes with the provided handler. +func (a *API) RegisterQueryAPI(handler http.Handler) { a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", handler, true, "POST") a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", handler, true, "GET", "POST") a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", handler, true, "GET", "POST") @@ -431,9 +314,21 @@ func (a *API) registerQueryAPI(handler http.Handler) { // RegisterQueryFrontend registers the Prometheus routes supported by the // Cortex querier service. Currently this can not be registered simultaneously // with the Querier. -func (a *API) RegisterQueryFrontend(f *frontend.Frontend) { - frontend.RegisterFrontendServer(a.server.GRPC, f) - a.registerQueryAPI(f.Handler()) +func (a *API) RegisterQueryFrontendHandler(h http.Handler) { + a.RegisterQueryAPI(h) +} + +func (a *API) RegisterQueryFrontend1(f *frontendv1.Frontend) { + frontendv1pb.RegisterFrontendServer(a.server.GRPC, f) +} + +func (a *API) RegisterQueryFrontend2(f *frontendv2.Frontend) { + frontendv2pb.RegisterFrontendForQuerierServer(a.server.GRPC, f) +} + +func (a *API) RegisterQueryScheduler(f *scheduler.Scheduler) { + schedulerpb.RegisterSchedulerForFrontendServer(a.server.GRPC, f) + schedulerpb.RegisterSchedulerForQuerierServer(a.server.GRPC, f) } // RegisterServiceMapHandler registers the Cortex structs service handler diff --git a/vendor/github.com/cortexproject/cortex/pkg/api/handlers.go b/vendor/github.com/cortexproject/cortex/pkg/api/handlers.go index 665d220b3964..9016a1f91348 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/api/handlers.go +++ b/vendor/github.com/cortexproject/cortex/pkg/api/handlers.go @@ -1,14 +1,34 @@ package api import ( + "context" "html/template" "net/http" "path" + "regexp" "sync" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/gorilla/mux" + "github.com/opentracing-contrib/go-stdlib/nethttp" + "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/route" + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/storage" + v1 "github.com/prometheus/prometheus/web/api/v1" + "github.com/weaveworks/common/instrument" + "github.com/weaveworks/common/middleware" "gopkg.in/yaml.v2" + "github.com/cortexproject/cortex/pkg/chunk/purger" + "github.com/cortexproject/cortex/pkg/distributor" + "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/util" ) @@ -109,3 +129,120 @@ func configHandler(cfg interface{}) http.HandlerFunc { } } } + +// NewQuerierHandler returns a HTTP handler that can be used by the querier service to +// either register with the frontend worker query processor or with the external HTTP +// server to fulfill the Prometheus query API. +func NewQuerierHandler( + cfg Config, + queryable storage.SampleAndChunkQueryable, + engine *promql.Engine, + distributor *distributor.Distributor, + tombstonesLoader *purger.TombstonesLoader, + reg prometheus.Registerer, + logger log.Logger, +) http.Handler { + // Prometheus histograms for requests to the querier. + querierRequestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_request_duration_seconds", + Help: "Time (in seconds) spent serving HTTP requests to the querier.", + Buckets: instrument.DefBuckets, + }, []string{"method", "route", "status_code", "ws"}) + + receivedMessageSize := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_request_message_bytes", + Help: "Size (in bytes) of messages received in the request to the querier.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + + sentMessageSize := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_response_message_bytes", + Help: "Size (in bytes) of messages sent in response by the querier.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + + inflightRequests := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "cortex", + Name: "querier_inflight_requests", + Help: "Current number of inflight requests to the querier.", + }, []string{"method", "route"}) + + api := v1.NewAPI( + engine, + errorTranslateQueryable{queryable}, // Translate errors to errors expected by API. + func(context.Context) v1.TargetRetriever { return &querier.DummyTargetRetriever{} }, + func(context.Context) v1.AlertmanagerRetriever { return &querier.DummyAlertmanagerRetriever{} }, + func() config.Config { return config.Config{} }, + map[string]string{}, // TODO: include configuration flags + v1.GlobalURLOptions{}, + func(f http.HandlerFunc) http.HandlerFunc { return f }, + nil, // Only needed for admin APIs. + "", // This is for snapshots, which is disabled when admin APIs are disabled. Hence empty. + false, // Disable admin APIs. + logger, + func(context.Context) v1.RulesRetriever { return &querier.DummyRulesRetriever{} }, + 0, 0, 0, // Remote read samples and concurrency limit. + regexp.MustCompile(".*"), + func() (v1.RuntimeInfo, error) { return v1.RuntimeInfo{}, errors.New("not implemented") }, + &v1.PrometheusVersion{}, + // This is used for the stats API which we should not support. Or find other ways to. + prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return nil, nil }), + ) + + router := mux.NewRouter() + + // Use a separate metric for the querier in order to differentiate requests from the query-frontend when + // running Cortex as a single binary. + inst := middleware.Instrument{ + RouteMatcher: router, + Duration: querierRequestDuration, + RequestBodySize: receivedMessageSize, + ResponseBodySize: sentMessageSize, + InflightRequests: inflightRequests, + } + cacheGenHeaderMiddleware := getHTTPCacheGenNumberHeaderSetterMiddleware(tombstonesLoader) + middlewares := middleware.Merge(inst, cacheGenHeaderMiddleware) + router.Use(middlewares.Wrap) + + // Define the prefixes for all routes + prefix := cfg.ServerPrefix + cfg.PrometheusHTTPPrefix + legacyPrefix := cfg.ServerPrefix + cfg.LegacyHTTPPrefix + + promRouter := route.New().WithPrefix(prefix + "/api/v1") + api.Register(promRouter) + + legacyPromRouter := route.New().WithPrefix(legacyPrefix + "/api/v1") + api.Register(legacyPromRouter) + + // TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: + // https://github.com/prometheus/prometheus/pull/7125/files + router.Path(prefix + "/api/v1/metadata").Handler(querier.MetadataHandler(distributor)) + router.Path(prefix + "/api/v1/read").Handler(querier.RemoteReadHandler(queryable)) + router.Path(prefix + "/api/v1/read").Methods("POST").Handler(promRouter) + router.Path(prefix+"/api/v1/query").Methods("GET", "POST").Handler(promRouter) + router.Path(prefix+"/api/v1/query_range").Methods("GET", "POST").Handler(promRouter) + router.Path(prefix+"/api/v1/labels").Methods("GET", "POST").Handler(promRouter) + router.Path(prefix + "/api/v1/label/{name}/values").Methods("GET").Handler(promRouter) + router.Path(prefix+"/api/v1/series").Methods("GET", "POST", "DELETE").Handler(promRouter) + router.Path(prefix + "/api/v1/metadata").Methods("GET").Handler(promRouter) + + // TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: + // https://github.com/prometheus/prometheus/pull/7125/files + router.Path(legacyPrefix + "/api/v1/metadata").Handler(querier.MetadataHandler(distributor)) + router.Path(legacyPrefix + "/api/v1/read").Handler(querier.RemoteReadHandler(queryable)) + router.Path(legacyPrefix + "/api/v1/read").Methods("POST").Handler(legacyPromRouter) + router.Path(legacyPrefix+"/api/v1/query").Methods("GET", "POST").Handler(legacyPromRouter) + router.Path(legacyPrefix+"/api/v1/query_range").Methods("GET", "POST").Handler(legacyPromRouter) + router.Path(legacyPrefix+"/api/v1/labels").Methods("GET", "POST").Handler(legacyPromRouter) + router.Path(legacyPrefix + "/api/v1/label/{name}/values").Methods("GET").Handler(legacyPromRouter) + router.Path(legacyPrefix+"/api/v1/series").Methods("GET", "POST", "DELETE").Handler(legacyPromRouter) + router.Path(legacyPrefix + "/api/v1/metadata").Methods("GET").Handler(legacyPromRouter) + + // Add a middleware to extract the trace context and add a header. + return nethttp.MiddlewareFunc(opentracing.GlobalTracer(), router.ServeHTTP, nethttp.OperationNameFunc(func(r *http.Request) string { + return "internalQuerier" + })) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go index a509c5bfb6e4..1e638f6091f0 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go @@ -34,6 +34,8 @@ type Config struct { SSL bool `yaml:"SSL"` HostVerification bool `yaml:"host_verification"` CAPath string `yaml:"CA_path"` + CertPath string `yaml:"tls_cert_path"` + KeyPath string `yaml:"tls_key_path"` Auth bool `yaml:"auth"` Username string `yaml:"username"` Password flagext.Secret `yaml:"password"` @@ -62,6 +64,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.SSL, "cassandra.ssl", false, "Use SSL when connecting to cassandra instances.") f.BoolVar(&cfg.HostVerification, "cassandra.host-verification", true, "Require SSL certificate validation.") f.StringVar(&cfg.CAPath, "cassandra.ca-path", "", "Path to certificate file to verify the peer.") + f.StringVar(&cfg.CertPath, "cassandra.tls-cert-path", "", "Path to certificate file used by TLS.") + f.StringVar(&cfg.KeyPath, "cassandra.tls-key-path", "", "Path to private key file used by TLS.") f.BoolVar(&cfg.Auth, "cassandra.auth", false, "Enable password authentication when connecting to cassandra.") f.StringVar(&cfg.Username, "cassandra.username", "", "Username to use when connecting to cassandra.") f.Var(&cfg.Password, "cassandra.password", "Password to use when connecting to cassandra.") @@ -86,6 +90,12 @@ func (cfg *Config) Validate() error { if cfg.SSL && cfg.HostVerification && len(strings.Split(cfg.Addresses, ",")) != 1 { return errors.Errorf("Host verification is only possible for a single host.") } + if cfg.SSL && cfg.CertPath != "" && cfg.KeyPath == "" { + return errors.Errorf("TLS certificate specified, but private key configuration is missing.") + } + if cfg.SSL && cfg.KeyPath != "" && cfg.CertPath == "" { + return errors.Errorf("TLS private key specified, but certificate configuration is missing.") + } return nil } @@ -144,17 +154,29 @@ func (cfg *Config) setClusterConfig(cluster *gocql.ClusterConfig) error { cluster.DisableInitialHostLookup = cfg.DisableInitialHostLookup if cfg.SSL { + tlsConfig := &tls.Config{} + + if cfg.CertPath != "" { + cert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) + if err != nil { + return errors.Wrap(err, "Unable to load TLS certificate and private key") + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + if cfg.HostVerification { + tlsConfig.ServerName = strings.Split(cfg.Addresses, ",")[0] + cluster.SslOpts = &gocql.SslOptions{ CaPath: cfg.CAPath, EnableHostVerification: true, - Config: &tls.Config{ - ServerName: strings.Split(cfg.Addresses, ",")[0], - }, + Config: tlsConfig, } } else { cluster.SslOpts = &gocql.SslOptions{ EnableHostVerification: false, + Config: tlsConfig, } } } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk.go index d52acf4bc4de..2879a8050ae1 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk.go @@ -20,15 +20,12 @@ import ( "github.com/cortexproject/cortex/pkg/prom1/storage/metric" ) -// Errors that decode can return const ( - ErrInvalidChecksum = errs.Error("invalid chunk checksum") - ErrWrongMetadata = errs.Error("wrong chunk metadata") - ErrMetadataLength = errs.Error("chunk metadata wrong length") - ErrDataLength = errs.Error("chunk data wrong length") - ErrSliceOutOfRange = errs.Error("chunk can't be sliced out of its data range") - ErrSliceNoDataInRange = errs.Error("chunk has no data for given range to slice") - ErrSliceChunkOverflow = errs.Error("slicing should not overflow a chunk") + ErrInvalidChecksum = errs.Error("invalid chunk checksum") + ErrWrongMetadata = errs.Error("wrong chunk metadata") + ErrMetadataLength = errs.Error("chunk metadata wrong length") + ErrDataLength = errs.Error("chunk data wrong length") + ErrSliceOutOfRange = errs.Error("chunk can't be sliced out of its data range") ) var castagnoliTable = crc32.MakeTable(crc32.Castagnoli) @@ -338,39 +335,11 @@ func (c *Chunk) Slice(from, through model.Time) (*Chunk, error) { return nil, ErrSliceOutOfRange } - itr := c.Data.NewIterator(nil) - if !itr.FindAtOrAfter(from) { - return nil, ErrSliceNoDataInRange - } - - pc, err := prom_chunk.NewForEncoding(c.Data.Encoding()) + pc, err := c.Data.Rebound(from, through) if err != nil { return nil, err } - for !itr.Value().Timestamp.After(through) { - oc, err := pc.Add(itr.Value()) - if err != nil { - return nil, err - } - - if oc != nil { - return nil, ErrSliceChunkOverflow - } - if !itr.Scan() { - break - } - } - - err = itr.Err() - if err != nil { - return nil, err - } - - if pc.Len() == 0 { - return nil, ErrSliceNoDataInRange - } - nc := NewChunk(c.UserID, c.Fingerprint, c.Metric, pc, from, through) return &nc, nil } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go index 2352938a8007..97670b10fa13 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -16,8 +17,10 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/cortexproject/cortex/pkg/chunk/cache" + "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/extract" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/spanlogger" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -74,11 +77,16 @@ func (cfg *StoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.WriteDedupeCacheConfig.RegisterFlagsWithPrefix("store.index-cache-write.", "Cache config for index entry writing. ", f) f.Var(&cfg.CacheLookupsOlderThan, "store.cache-lookups-older-than", "Cache index entries older than this period. 0 to disable.") - f.Var(&cfg.MaxLookBackPeriod, "store.max-look-back-period", "Limit how long back data can be queried") + f.Var(&cfg.MaxLookBackPeriod, "store.max-look-back-period", "Deprecated: use -querier.max-query-lookback instead. Limit how long back data can be queried. This setting applies to chunks storage only.") // To be removed in Cortex 1.8. } // Validate validates the store config. -func (cfg *StoreConfig) Validate() error { +func (cfg *StoreConfig) Validate(logger log.Logger) error { + if cfg.MaxLookBackPeriod > 0 { + flagext.DeprecatedFlagsUsed.Inc() + level.Warn(logger).Log("msg", "running with DEPRECATED flag -store.max-look-back-period, use -querier.max-query-lookback instead.") + } + if err := cfg.ChunkCacheConfig.Validate(); err != nil { return err } @@ -677,7 +685,7 @@ func (c *baseStore) reboundChunk(ctx context.Context, userID, chunkID string, pa var newChunks []*Chunk if partiallyDeletedInterval.Start > chunk.From { newChunk, err := chunk.Slice(chunk.From, partiallyDeletedInterval.Start-1) - if err != nil && err != ErrSliceNoDataInRange { + if err != nil && err != encoding.ErrSliceNoDataInRange { return errors.Wrapf(err, "when slicing chunk for interval %d - %d", chunk.From, partiallyDeletedInterval.Start-1) } @@ -688,7 +696,7 @@ func (c *baseStore) reboundChunk(ctx context.Context, userID, chunkID string, pa if partiallyDeletedInterval.End < chunk.Through { newChunk, err := chunk.Slice(partiallyDeletedInterval.End+1, chunk.Through) - if err != nil && err != ErrSliceNoDataInRange { + if err != nil && err != encoding.ErrSliceNoDataInRange { return errors.Wrapf(err, "when slicing chunk for interval %d - %d", partiallyDeletedInterval.End+1, chunk.Through) } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/bigchunk.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/bigchunk.go index 8683ebc5a00b..c05defedb172 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/bigchunk.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/bigchunk.go @@ -210,6 +210,10 @@ func (b *bigchunk) Slice(start, end model.Time) Chunk { } } +func (b *bigchunk) Rebound(start, end model.Time) (Chunk, error) { + return reboundChunk(b, start, end) +} + type writer struct { io.Writer } diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/chunk.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/chunk.go index b31304714d1b..97c95e41a773 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/chunk.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/chunk.go @@ -22,12 +22,18 @@ import ( "sort" "github.com/prometheus/common/model" + errs "github.com/weaveworks/common/errors" "github.com/cortexproject/cortex/pkg/prom1/storage/metric" ) -// ChunkLen is the length of a chunk in bytes. -const ChunkLen = 1024 +const ( + // ChunkLen is the length of a chunk in bytes. + ChunkLen = 1024 + + ErrSliceNoDataInRange = errs.Error("chunk has no data for given range to slice") + ErrSliceChunkOverflow = errs.Error("slicing should not overflow a chunk") +) var ( errChunkBoundsExceeded = errors.New("attempted access outside of chunk boundaries") @@ -50,10 +56,15 @@ type Chunk interface { Encoding() Encoding Utilization() float64 - // Slice returns a smaller chunk the includes all samples between start and end + // Slice returns a smaller chunk that includes all samples between start and end // (inclusive). Its may over estimate. On some encodings it is a noop. Slice(start, end model.Time) Chunk + // Rebound returns a smaller chunk that includes all samples between start and end (inclusive). + // We do not want to change existing Slice implementations because + // it is built specifically for query optimization and is a noop for some of the encodings. + Rebound(start, end model.Time) (Chunk, error) + // Len returns the number of samples in the chunk. Implementations may be // expensive. Len() int @@ -246,3 +257,40 @@ func (it *indexAccessingChunkIterator) Batch(size int) Batch { func (it *indexAccessingChunkIterator) Err() error { return it.acc.err() } + +func reboundChunk(c Chunk, start, end model.Time) (Chunk, error) { + itr := c.NewIterator(nil) + if !itr.FindAtOrAfter(start) { + return nil, ErrSliceNoDataInRange + } + + pc, err := NewForEncoding(c.Encoding()) + if err != nil { + return nil, err + } + + for !itr.Value().Timestamp.After(end) { + oc, err := pc.Add(itr.Value()) + if err != nil { + return nil, err + } + + if oc != nil { + return nil, ErrSliceChunkOverflow + } + if !itr.Scan() { + break + } + } + + err = itr.Err() + if err != nil { + return nil, err + } + + if pc.Len() == 0 { + return nil, ErrSliceNoDataInRange + } + + return pc, nil +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/doubledelta.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/doubledelta.go index 683ce844eef6..e0e43e7d63bd 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/doubledelta.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/doubledelta.go @@ -233,6 +233,10 @@ func (c *doubleDeltaEncodedChunk) Slice(_, _ model.Time) Chunk { return c } +func (c *doubleDeltaEncodedChunk) Rebound(start, end model.Time) (Chunk, error) { + return reboundChunk(c, start, end) +} + // Marshal implements chunk. func (c doubleDeltaEncodedChunk) Marshal(w io.Writer) error { if len(c) > math.MaxUint16 { diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go index a9d1c2f28771..fe67337ecadf 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/encoding/varbit.go @@ -287,6 +287,10 @@ func (c *varbitChunk) Slice(_, _ model.Time) Chunk { return c } +func (c *varbitChunk) Rebound(start, end model.Time) (Chunk, error) { + return reboundChunk(c, start, end) +} + // Marshal implements chunk. func (c varbitChunk) Marshal(w io.Writer) error { size := c.Size() diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/grpc/grpc_client.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/grpc/grpc_client.go index 7c9998f657f6..5e7b635be71b 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/grpc/grpc_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/grpc/grpc_client.go @@ -22,7 +22,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { func connectToGrpcServer(serverAddress string) (GrpcStoreClient, *grpc.ClientConn, error) { params := keepalive.ClientParameters{ Time: time.Second * 20, - Timeout: time.Minute * 10, + Timeout: time.Second * 10, PermitWithoutStream: true, } param := grpc.WithKeepaliveParams(params) diff --git a/vendor/github.com/cortexproject/cortex/pkg/chunk/table_manager.go b/vendor/github.com/cortexproject/cortex/pkg/chunk/table_manager.go index 7426f19f3213..eda8a83f753f 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/chunk/table_manager.go +++ b/vendor/github.com/cortexproject/cortex/pkg/chunk/table_manager.go @@ -14,7 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" + tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/weaveworks/common/instrument" "github.com/weaveworks/common/mtime" @@ -141,7 +141,7 @@ func (cfg *TableManagerConfig) Validate() error { func (cfg *TableManagerConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.ThroughputUpdatesDisabled, "table-manager.throughput-updates-disabled", false, "If true, disable all changes to DB capacity") f.BoolVar(&cfg.RetentionDeletesEnabled, "table-manager.retention-deletes-enabled", false, "If true, enables retention deletes of DB tables") - f.Var(&cfg.RetentionPeriodModel, "table-manager.retention-period", "Tables older than this retention period are deleted. Note: This setting is destructive to data!(default: 0, which disables deletion)") + f.Var(&cfg.RetentionPeriodModel, "table-manager.retention-period", "Tables older than this retention period are deleted. Must be either 0 (disabled) or a multiple of 24h. When enabled, be aware this setting is destructive to data!") f.DurationVar(&cfg.PollInterval, "table-manager.poll-interval", 2*time.Minute, "How frequently to poll backend to learn our capacity.") f.DurationVar(&cfg.CreationGracePeriod, "table-manager.periodic-table.grace-period", 10*time.Minute, "Periodic tables grace period (duration which table will be created/deleted before/after it's needed).") @@ -470,7 +470,7 @@ func (m *TableManager) partitionTables(ctx context.Context, descriptions []Table func (m *TableManager) createTables(ctx context.Context, descriptions []TableDesc) error { numFailures := 0 - merr := tsdberrors.MultiError{} + merr := tsdb_errors.NewMulti() for _, desc := range descriptions { level.Info(util.Logger).Log("msg", "creating table", "table", desc.Name) @@ -487,7 +487,7 @@ func (m *TableManager) createTables(ctx context.Context, descriptions []TableDes func (m *TableManager) deleteTables(ctx context.Context, descriptions []TableDesc) error { numFailures := 0 - merr := tsdberrors.MultiError{} + merr := tsdb_errors.NewMulti() for _, desc := range descriptions { level.Info(util.Logger).Log("msg", "table has exceeded the retention period", "table", desc.Name) diff --git a/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go b/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go index 411f66d336e6..981a83f22c0c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go +++ b/vendor/github.com/cortexproject/cortex/pkg/compactor/blocks_cleaner.go @@ -120,7 +120,7 @@ func (c *BlocksCleaner) cleanUsers(ctx context.Context) error { return errors.Wrap(err, "failed to discover users from bucket") } - errs := tsdb_errors.MultiError{} + errs := tsdb_errors.NewMulti() for _, userID := range users { // Ensure the context has not been canceled (ie. shutdown has been triggered). if ctx.Err() != nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go b/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go index f6f08dcf3fbb..c1f6478fc816 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go +++ b/vendor/github.com/cortexproject/cortex/pkg/compactor/compactor.go @@ -24,6 +24,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring" cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -39,6 +40,9 @@ type Config struct { CompactionConcurrency int `yaml:"compaction_concurrency"` DeletionDelay time.Duration `yaml:"deletion_delay"` + EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"` + DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"` + // Compactors sharding. ShardingEnabled bool `yaml:"sharding_enabled"` ShardingRing RingConfig `yaml:"sharding_ring"` @@ -71,6 +75,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { "If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. "+ "If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, "+ "if store gateway still has the block loaded, or compactor is ignoring the deletion because it's compacting the block at the same time.") + + f.Var(&cfg.EnabledTenants, "compactor.enabled-tenants", "Comma separated list of tenants that can be compacted. If specified, only these tenants will be compacted by compactor, otherwise all tenants can be compacted. Subject to sharding.") + f.Var(&cfg.DisabledTenants, "compactor.disabled-tenants", "Comma separated list of tenants that cannot be compacted by this compactor. If specified, and compactor would normally pick given tenant for compaction (via -compactor.enabled-tenants or sharding), it will be ignored instead.") } // Compactor is a multi-tenant TSDB blocks compactor based on Thanos. @@ -83,6 +90,12 @@ type Compactor struct { parentLogger log.Logger registerer prometheus.Registerer + // If empty, all users are enabled. If not empty, only users in the map are enabled (possibly owned by compactor, also subject to sharding configuration). + enabledUsers map[string]struct{} + + // If empty, no users are disabled. If not empty, users in the map are disabled (not owned by this compactor). + disabledUsers map[string]struct{} + // Function that creates bucket client and TSDB compactor using the context. // Useful for injecting mock objects from tests. createBucketClientAndTsdbCompactor func(ctx context.Context) (objstore.Bucket, tsdb.Compactor, error) @@ -179,6 +192,24 @@ func newCompactor( }), } + if len(compactorCfg.EnabledTenants) > 0 { + c.enabledUsers = map[string]struct{}{} + for _, u := range compactorCfg.EnabledTenants { + c.enabledUsers[u] = struct{}{} + } + + level.Info(c.logger).Log("msg", "using enabled users", "enabled", strings.Join(compactorCfg.EnabledTenants, ", ")) + } + + if len(compactorCfg.DisabledTenants) > 0 { + c.disabledUsers = map[string]struct{}{} + for _, u := range compactorCfg.DisabledTenants { + c.disabledUsers[u] = struct{}{} + } + + level.Info(c.logger).Log("msg", "using disabled users", "disabled", strings.Join(compactorCfg.DisabledTenants, ", ")) + } + c.Service = services.NewBasicService(c.starting, c.running, c.stopping) return c, nil @@ -313,7 +344,7 @@ func (c *Compactor) compactUsers(ctx context.Context) error { } level.Info(c.logger).Log("msg", "discovered users from bucket", "users", len(users)) - errs := tsdb_errors.MultiError{} + errs := tsdb_errors.NewMulti() for _, userID := range users { // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). @@ -322,15 +353,13 @@ func (c *Compactor) compactUsers(ctx context.Context) error { return ctx.Err() } - // If sharding is enabled, ensure the user ID belongs to our shard. - if c.compactorCfg.ShardingEnabled { - if owned, err := c.ownUser(userID); err != nil { - level.Warn(c.logger).Log("msg", "unable to check if user is owned by this shard", "user", userID, "err", err) - continue - } else if !owned { - level.Debug(c.logger).Log("msg", "skipping user because not owned by this shard", "user", userID) - continue - } + // Ensure the user ID belongs to our shard. + if owned, err := c.ownUser(userID); err != nil { + level.Warn(c.logger).Log("msg", "unable to check if user is owned by this shard", "user", userID, "err", err) + continue + } else if !owned { + level.Debug(c.logger).Log("msg", "skipping user because not owned by this shard", "user", userID) + continue } level.Info(c.logger).Log("msg", "starting compaction of user blocks", "user", userID) @@ -444,6 +473,10 @@ func (c *Compactor) discoverUsers(ctx context.Context) ([]string, error) { } func (c *Compactor) ownUser(userID string) (bool, error) { + if !isAllowedUser(c.enabledUsers, c.disabledUsers, userID) { + return false, nil + } + // Always owned if sharding is disabled. if !c.compactorCfg.ShardingEnabled { return true, nil @@ -455,7 +488,7 @@ func (c *Compactor) ownUser(userID string) (bool, error) { userHash := hasher.Sum32() // Check whether this compactor instance owns the user. - rs, err := c.ring.Get(userHash, ring.Read, []ring.IngesterDesc{}) + rs, err := c.ring.Get(userHash, ring.Compactor, []ring.IngesterDesc{}) if err != nil { return false, err } @@ -466,3 +499,19 @@ func (c *Compactor) ownUser(userID string) (bool, error) { return rs.Ingesters[0].Addr == c.ringLifecycler.Addr, nil } + +func isAllowedUser(enabledUsers, disabledUsers map[string]struct{}, userID string) bool { + if len(enabledUsers) > 0 { + if _, ok := enabledUsers[userID]; !ok { + return false + } + } + + if len(disabledUsers) > 0 { + if _, ok := disabledUsers[userID]; ok { + return false + } + } + + return true +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go b/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go index 336232f26cf0..4aba510a2f3a 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go +++ b/vendor/github.com/cortexproject/cortex/pkg/cortex/cortex.go @@ -12,6 +12,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "github.com/prometheus/prometheus/promql" + prom_storage "github.com/prometheus/prometheus/storage" "github.com/weaveworks/common/server" "github.com/weaveworks/common/signals" "google.golang.org/grpc/health/grpc_health_v1" @@ -20,7 +22,6 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager" "github.com/cortexproject/cortex/pkg/api" "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/cache" "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/chunk/storage" @@ -31,15 +32,18 @@ import ( "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/flusher" + "github.com/cortexproject/cortex/pkg/frontend" + frontendv1 "github.com/cortexproject/cortex/pkg/frontend/v1" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/querier" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/querier/queryrange" + querier_worker "github.com/cortexproject/cortex/pkg/querier/worker" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/memberlist" "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/ruler/rules" + "github.com/cortexproject/cortex/pkg/scheduler" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util" @@ -76,33 +80,34 @@ type Config struct { PrintConfig bool `yaml:"-"` HTTPPrefix string `yaml:"http_prefix"` - API api.Config `yaml:"api"` - Server server.Config `yaml:"server"` - Distributor distributor.Config `yaml:"distributor"` - Querier querier.Config `yaml:"querier"` - IngesterClient client.Config `yaml:"ingester_client"` - Ingester ingester.Config `yaml:"ingester"` - Flusher flusher.Config `yaml:"flusher"` - Storage storage.Config `yaml:"storage"` - ChunkStore chunk.StoreConfig `yaml:"chunk_store"` - Schema chunk.SchemaConfig `yaml:"schema" doc:"hidden"` // Doc generation tool doesn't support it because part of the SchemaConfig doesn't support CLI flags (needs manual documentation) - LimitsConfig validation.Limits `yaml:"limits"` - Prealloc client.PreallocConfig `yaml:"prealloc" doc:"hidden"` - Worker frontend.WorkerConfig `yaml:"frontend_worker"` - Frontend frontend.Config `yaml:"frontend"` - QueryRange queryrange.Config `yaml:"query_range"` - TableManager chunk.TableManagerConfig `yaml:"table_manager"` - Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. - BlocksStorage tsdb.BlocksStorageConfig `yaml:"blocks_storage"` - Compactor compactor.Config `yaml:"compactor"` - StoreGateway storegateway.Config `yaml:"store_gateway"` - PurgerConfig purger.Config `yaml:"purger"` - - Ruler ruler.Config `yaml:"ruler"` - Configs configs.Config `yaml:"configs"` - Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager"` - RuntimeConfig runtimeconfig.ManagerConfig `yaml:"runtime_config"` - MemberlistKV memberlist.KVConfig `yaml:"memberlist"` + API api.Config `yaml:"api"` + Server server.Config `yaml:"server"` + Distributor distributor.Config `yaml:"distributor"` + Querier querier.Config `yaml:"querier"` + IngesterClient client.Config `yaml:"ingester_client"` + Ingester ingester.Config `yaml:"ingester"` + Flusher flusher.Config `yaml:"flusher"` + Storage storage.Config `yaml:"storage"` + ChunkStore chunk.StoreConfig `yaml:"chunk_store"` + Schema chunk.SchemaConfig `yaml:"schema" doc:"hidden"` // Doc generation tool doesn't support it because part of the SchemaConfig doesn't support CLI flags (needs manual documentation) + LimitsConfig validation.Limits `yaml:"limits"` + Prealloc client.PreallocConfig `yaml:"prealloc" doc:"hidden"` + Worker querier_worker.Config `yaml:"frontend_worker"` + Frontend frontend.CombinedFrontendConfig `yaml:"frontend"` + QueryRange queryrange.Config `yaml:"query_range"` + TableManager chunk.TableManagerConfig `yaml:"table_manager"` + Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. + BlocksStorage tsdb.BlocksStorageConfig `yaml:"blocks_storage"` + Compactor compactor.Config `yaml:"compactor"` + StoreGateway storegateway.Config `yaml:"store_gateway"` + PurgerConfig purger.Config `yaml:"purger"` + + Ruler ruler.Config `yaml:"ruler"` + Configs configs.Config `yaml:"configs"` + Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager"` + RuntimeConfig runtimeconfig.ManagerConfig `yaml:"runtime_config"` + MemberlistKV memberlist.KVConfig `yaml:"memberlist"` + QueryScheduler scheduler.Config `yaml:"query_scheduler"` } // RegisterFlags registers flag. @@ -148,6 +153,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Alertmanager.RegisterFlags(f) c.RuntimeConfig.RegisterFlags(f) c.MemberlistKV.RegisterFlags(f, "") + c.QueryScheduler.RegisterFlags(f) // These don't seem to have a home. f.IntVar(&chunk_util.QueryParallelism, "querier.query-parallelism", 100, "Max subqueries run in parallel per higher-level query.") @@ -169,7 +175,7 @@ func (c *Config) Validate(log log.Logger) error { if err := c.Storage.Validate(); err != nil { return errors.Wrap(err, "invalid storage config") } - if err := c.ChunkStore.Validate(); err != nil { + if err := c.ChunkStore.Validate(log); err != nil { return errors.Wrap(err, "invalid chunk store config") } if err := c.Ruler.Validate(c.LimitsConfig); err != nil { @@ -249,21 +255,23 @@ type Cortex struct { ServiceMap map[string]services.Service ModuleManager *modules.Manager - API *api.API - Server *server.Server - Ring *ring.Ring - Overrides *validation.Overrides - Distributor *distributor.Distributor - Ingester *ingester.Ingester - Flusher *flusher.Flusher - Store chunk.Store - DeletesStore *purger.DeleteStore - Frontend *frontend.Frontend - TableManager *chunk.TableManager - Cache cache.Cache - RuntimeConfig *runtimeconfig.Manager - Purger *purger.Purger - TombstonesLoader *purger.TombstonesLoader + API *api.API + Server *server.Server + Ring *ring.Ring + Overrides *validation.Overrides + Distributor *distributor.Distributor + Ingester *ingester.Ingester + Flusher *flusher.Flusher + Store chunk.Store + DeletesStore *purger.DeleteStore + Frontend *frontendv1.Frontend + TableManager *chunk.TableManager + RuntimeConfig *runtimeconfig.Manager + Purger *purger.Purger + TombstonesLoader *purger.TombstonesLoader + QuerierQueryable prom_storage.SampleAndChunkQueryable + QuerierEngine *promql.Engine + QueryFrontendTripperware queryrange.Tripperware Ruler *ruler.Ruler RulerStorage rules.RuleStore @@ -290,11 +298,15 @@ func New(cfg Config) (*Cortex, error) { // Don't check auth header on TransferChunks, as we weren't originally // sending it and this could cause transfers to fail on update. - // - // Also don't check auth /frontend.Frontend/Process, as this handles - // queries for multiple users. cfg.API.HTTPAuthMiddleware = fakeauth.SetupAuthMiddleware(&cfg.Server, cfg.AuthEnabled, - []string{"/cortex.Ingester/TransferChunks", "/frontend.Frontend/Process"}) + // Also don't check auth for these gRPC methods, since single call is used for multiple users (or no user like health check). + []string{ + "/grpc.health.v1.Health/Check", + "/cortex.Ingester/TransferChunks", + "/frontend.Frontend/Process", + "/schedulerpb.SchedulerForFrontend/FrontendLoop", + "/schedulerpb.SchedulerForQuerier/QuerierLoop", + }) cortex := &Cortex{ Cfg: cfg, diff --git a/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go b/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go index 0bf3ea5c13a3..91336b1b0bc4 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go +++ b/vendor/github.com/cortexproject/cortex/pkg/cortex/modules.go @@ -5,15 +5,14 @@ import ( "os" "time" + "github.com/NYTimes/gziphandler" "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" prom_storage "github.com/prometheus/prometheus/storage" httpgrpc_server "github.com/weaveworks/common/httpgrpc/server" - "github.com/weaveworks/common/instrument" - "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" "github.com/cortexproject/cortex/pkg/alertmanager" @@ -26,14 +25,17 @@ import ( "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/flusher" + frontend "github.com/cortexproject/cortex/pkg/frontend" + "github.com/cortexproject/cortex/pkg/frontend/transport" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/querier" - "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/querier/queryrange" + querier_worker "github.com/cortexproject/cortex/pkg/querier/worker" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" "github.com/cortexproject/cortex/pkg/ring/kv/memberlist" "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/scheduler" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" @@ -45,31 +47,34 @@ import ( // The various modules that make up Cortex. const ( - API string = "api" - Ring string = "ring" - RuntimeConfig string = "runtime-config" - Overrides string = "overrides" - Server string = "server" - Distributor string = "distributor" - DistributorService string = "distributor-service" - Ingester string = "ingester" - IngesterService string = "ingester-service" - Flusher string = "flusher" - Querier string = "querier" - StoreQueryable string = "store-queryable" - QueryFrontend string = "query-frontend" - Store string = "store" - DeleteRequestsStore string = "delete-requests-store" - TableManager string = "table-manager" - RulerStorage string = "ruler-storage" - Ruler string = "ruler" - Configs string = "configs" - AlertManager string = "alertmanager" - Compactor string = "compactor" - StoreGateway string = "store-gateway" - MemberlistKV string = "memberlist-kv" - Purger string = "purger" - All string = "all" + API string = "api" + Ring string = "ring" + RuntimeConfig string = "runtime-config" + Overrides string = "overrides" + Server string = "server" + Distributor string = "distributor" + DistributorService string = "distributor-service" + Ingester string = "ingester" + IngesterService string = "ingester-service" + Flusher string = "flusher" + Querier string = "querier" + Queryable string = "queryable" + StoreQueryable string = "store-queryable" + QueryFrontend string = "query-frontend" + QueryFrontendTripperware string = "query-frontend-tripperware" + Store string = "store" + DeleteRequestsStore string = "delete-requests-store" + TableManager string = "table-manager" + RulerStorage string = "ruler-storage" + Ruler string = "ruler" + Configs string = "configs" + AlertManager string = "alertmanager" + Compactor string = "compactor" + StoreGateway string = "store-gateway" + MemberlistKV string = "memberlist-kv" + Purger string = "purger" + QueryScheduler string = "query-scheduler" + All string = "all" ) func (t *Cortex) initAPI() (services.Service, error) { @@ -190,58 +195,116 @@ func (t *Cortex) initDistributor() (serv services.Service, err error) { return nil, nil } -func (t *Cortex) initQuerier() (serv services.Service, err error) { +// initQueryable instantiates the queryable and promQL engine used to service queries to +// Cortex. It also registers the API endpoints associated with those two services. +func (t *Cortex) initQueryable() (serv services.Service, err error) { querierRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "querier"}, prometheus.DefaultRegisterer) - queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, querierRegisterer) - - // Prometheus histograms for requests to the querier. - querierRequestDuration := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "querier_request_duration_seconds", - Help: "Time (in seconds) spent serving HTTP requests to the querier.", - Buckets: instrument.DefBuckets, - }, []string{"method", "route", "status_code", "ws"}) - - receivedMessageSize := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "querier_request_message_bytes", - Help: "Size (in bytes) of messages received in the request to the querier.", - Buckets: middleware.BodySizeBuckets, - }, []string{"method", "route"}) - - sentMessageSize := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "querier_response_message_bytes", - Help: "Size (in bytes) of messages sent in response by the querier.", - Buckets: middleware.BodySizeBuckets, - }, []string{"method", "route"}) - - inflightRequests := promauto.With(prometheus.DefaultRegisterer).NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "querier_inflight_requests", - Help: "Current number of inflight requests to the querier.", - }, []string{"method", "route"}) - - // if we are not configured for single binary mode then the querier needs to register its paths externally - registerExternally := !t.Cfg.isModuleEnabled(All) - handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration, receivedMessageSize, sentMessageSize, inflightRequests) - - // single binary mode requires a properly configured worker. if the operator did not attempt to configure the - // worker we will attempt an automatic configuration here - if t.Cfg.Worker.Address == "" && t.Cfg.isModuleEnabled(All) { - address := fmt.Sprintf("127.0.0.1:%d", t.Cfg.Server.GRPCListenPort) - level.Warn(util.Logger).Log("msg", "Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) - t.Cfg.Worker.Address = address - } - - // Query frontend worker will only be started after all its dependencies are started, not here. - // Worker may also be nil, if not configured, which is OK. - worker, err := frontend.NewWorker(t.Cfg.Worker, t.Cfg.Querier, httpgrpc_server.NewServer(handler), util.Logger) - if err != nil { - return + + // Create a querier queryable and PromQL engine + t.QuerierQueryable, t.QuerierEngine = querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, querierRegisterer) + + // Register the default endpoints that are always enabled for the querier module + t.API.RegisterQueryable(t.QuerierQueryable, t.Distributor) + + return nil, nil +} + +// initQuerier registers an internal HTTP router with a Prometheus API backed by the +// Cortex Queryable. Then it does one of the following: +// +// 1. Query-Frontend Enabled: If Cortex has an All or QueryFrontend target, the internal +// HTTP router is wrapped with Tenant ID parsing middleware and passed to the frontend +// worker. +// +// 2. Querier Standalone: The querier will register the internal HTTP router with the external +// HTTP router for the Prometheus API routes. Then the external HTTP server will be passed +// as a http.Handler to the frontend worker. +// +// Route Diagram: +// +// │ query +// │ request +// │ +// ▼ +// ┌──────────────────┐ QF to ┌──────────────────┐ +// │ external HTTP │ Worker │ │ +// │ router │──────────────▶│ frontend worker │ +// │ │ │ │ +// └──────────────────┘ └──────────────────┘ +// │ │ +// │ +// only in │ │ +// microservice ┌──────────────────┐ │ +// querier │ │ internal Querier │ │ +// ─ ─ ─ ─▶│ router │◀─────┘ +// │ │ +// └──────────────────┘ +// │ +// │ +// /metadata & /chunk ┌─────────────────────┼─────────────────────┐ +// requests │ │ │ +// │ │ │ +// ▼ ▼ ▼ +// ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ +// │ │ │ │ │ │ +// │Querier Queryable │ │ /api/v1 router │ │ /api/prom router │ +// │ │ │ │ │ │ +// └──────────────────┘ └──────────────────┘ └──────────────────┘ +// ▲ │ │ +// │ └──────────┬──────────┘ +// │ ▼ +// │ ┌──────────────────┐ +// │ │ │ +// └──────────────────────│ Prometheus API │ +// │ │ +// └──────────────────┘ +// +func (t *Cortex) initQuerier() (serv services.Service, err error) { + // Create a internal HTTP handler that is configured with the Prometheus API routes and points + // to a Prometheus API struct instantiated with the Cortex Queryable. + internalQuerierRouter := api.NewQuerierHandler( + t.Cfg.API, + t.QuerierQueryable, + t.QuerierEngine, + t.Distributor, + t.TombstonesLoader, + prometheus.DefaultRegisterer, + util.Logger, + ) + + // If the querier is running standalone without the query-frontend or query-scheduler, we must register it's internal + // HTTP handler externally and provide the external Cortex Server HTTP handler to the frontend worker + // to ensure requests it processes use the default middleware instrumentation. + if !t.Cfg.isModuleEnabled(QueryFrontend) && !t.Cfg.isModuleEnabled(QueryScheduler) && !t.Cfg.isModuleEnabled(All) { + // First, register the internal querier handler with the external HTTP server + t.API.RegisterQueryAPI(internalQuerierRouter) + + // Second, set the http.Handler that the frontend worker will use to process requests to point to + // the external HTTP server. This will allow the querier to consolidate query metrics both external + // and internal using the default instrumentation when running as a standalone service. + internalQuerierRouter = t.Server.HTTPServer.Handler + } else { + // Single binary mode requires a query frontend endpoint for the worker. If no frontend or scheduler endpoint + // is configured, Cortex will default to using frontend on localhost on it's own GRPC listening port. + if t.Cfg.Worker.FrontendAddress == "" || t.Cfg.Worker.SchedulerAddress == "" { + address := fmt.Sprintf("127.0.0.1:%d", t.Cfg.Server.GRPCListenPort) + level.Warn(util.Logger).Log("msg", "Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) + t.Cfg.Worker.FrontendAddress = address + } + + // If queries are processed using the external HTTP Server, we need wrap the internal querier with + // HTTP router with middleware to parse the tenant ID from the HTTP header and inject it into the + // request context. + internalQuerierRouter = t.API.AuthMiddleware.Wrap(internalQuerierRouter) + } + + // If neither frontend address or scheduler address is configured, no worker is needed. + if t.Cfg.Worker.FrontendAddress == "" && t.Cfg.Worker.SchedulerAddress == "" { + return nil, nil } - return worker, nil + t.Cfg.Worker.MaxConcurrentRequests = t.Cfg.Querier.MaxConcurrent + return querier_worker.NewQuerierWorker(t.Cfg.Worker, httpgrpc_server.NewServer(internalQuerierRouter), util.Logger, prometheus.DefaultRegisterer) } func (t *Cortex) initStoreQueryables() (services.Service, error) { @@ -399,20 +462,17 @@ func (t *Cortex) initDeleteRequestsStore() (serv services.Service, err error) { return } -func (t *Cortex) initQueryFrontend() (serv services.Service, err error) { +// initQueryFrontendTripperware instantiates the tripperware used by the query frontend +// to optimize Prometheus query requests. +func (t *Cortex) initQueryFrontendTripperware() (serv services.Service, err error) { // Load the schema only if sharded queries is set. if t.Cfg.QueryRange.ShardedQueries { - err = t.Cfg.Schema.Load() + err := t.Cfg.Schema.Load() if err != nil { - return + return nil, err } } - t.Frontend, err = frontend.New(t.Cfg.Frontend, t.Overrides, util.Logger, prometheus.DefaultRegisterer) - if err != nil { - return - } - tripperware, cache, err := queryrange.NewTripperware( t.Cfg.QueryRange, util.Logger, @@ -437,21 +497,51 @@ func (t *Cortex) initQueryFrontend() (serv services.Service, err error) { if err != nil { return nil, err } - t.Cache = cache - t.Frontend.Wrap(tripperware) - t.API.RegisterQueryFrontend(t.Frontend) + t.QueryFrontendTripperware = tripperware return services.NewIdleService(nil, func(_ error) error { - t.Frontend.Close() - if t.Cache != nil { - t.Cache.Stop() - t.Cache = nil + if cache != nil { + cache.Stop() + cache = nil } return nil }), nil } +func (t *Cortex) initQueryFrontend() (serv services.Service, err error) { + roundTripper, frontendV1, frontendV2, err := frontend.InitFrontend(t.Cfg.Frontend, t.Overrides, t.Cfg.Server.GRPCListenPort, util.Logger, prometheus.DefaultRegisterer) + if err != nil { + return nil, err + } + + // Wrap roundtripper into Tripperware. + roundTripper = t.QueryFrontendTripperware(roundTripper) + + handler := transport.NewHandler(t.Cfg.Frontend.Handler, roundTripper, util.Logger) + if t.Cfg.Frontend.CompressResponses { + handler = gziphandler.GzipHandler(handler) + } + + t.API.RegisterQueryFrontendHandler(handler) + + if frontendV1 != nil { + t.API.RegisterQueryFrontend1(frontendV1) + t.Frontend = frontendV1 + + return services.NewIdleService(nil, func(_ error) error { + frontendV1.Close() + return nil + }), nil + } else if frontendV2 != nil { + t.API.RegisterQueryFrontend2(frontendV2) + + return frontendV2, nil + } + + return nil, nil +} + func (t *Cortex) initTableManager() (services.Service, error) { if t.Cfg.Storage.Engine == storage.StorageEngineBlocks { return nil, nil // table manager isn't used in v2 @@ -655,6 +745,16 @@ func (t *Cortex) initPurger() (services.Service, error) { return t.Purger, nil } +func (t *Cortex) initQueryScheduler() (services.Service, error) { + s, err := scheduler.NewScheduler(t.Cfg.QueryScheduler, t.Overrides, util.Logger, prometheus.DefaultRegisterer) + if err != nil { + return nil, errors.Wrap(err, "query-scheduler init") + } + + t.API.RegisterQueryScheduler(s) + return s, nil +} + func (t *Cortex) setupModuleManager() error { mm := modules.NewManager() @@ -673,8 +773,10 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(Ingester, t.initIngester) mm.RegisterModule(IngesterService, t.initIngesterService, modules.UserInvisibleModule) mm.RegisterModule(Flusher, t.initFlusher) + mm.RegisterModule(Queryable, t.initQueryable, modules.UserInvisibleModule) mm.RegisterModule(Querier, t.initQuerier) mm.RegisterModule(StoreQueryable, t.initStoreQueryables, modules.UserInvisibleModule) + mm.RegisterModule(QueryFrontendTripperware, t.initQueryFrontendTripperware, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontend, t.initQueryFrontend) mm.RegisterModule(TableManager, t.initTableManager) mm.RegisterModule(RulerStorage, t.initRulerStorage, modules.UserInvisibleModule) @@ -684,30 +786,34 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(Compactor, t.initCompactor) mm.RegisterModule(StoreGateway, t.initStoreGateway) mm.RegisterModule(Purger, t.initPurger) + mm.RegisterModule(QueryScheduler, t.initQueryScheduler) mm.RegisterModule(All, nil) // Add dependencies deps := map[string][]string{ - API: {Server}, - Ring: {API, RuntimeConfig, MemberlistKV}, - Overrides: {RuntimeConfig}, - Distributor: {DistributorService, API}, - DistributorService: {Ring, Overrides}, - Store: {Overrides, DeleteRequestsStore}, - Ingester: {IngesterService, API}, - IngesterService: {Overrides, Store, RuntimeConfig, MemberlistKV}, - Flusher: {Store, API}, - Querier: {Overrides, DistributorService, Store, Ring, API, StoreQueryable, MemberlistKV}, - StoreQueryable: {Overrides, Store, MemberlistKV}, - QueryFrontend: {API, Overrides, DeleteRequestsStore}, - TableManager: {API}, - Ruler: {Overrides, DistributorService, Store, StoreQueryable, RulerStorage}, - Configs: {API}, - AlertManager: {API}, - Compactor: {API, MemberlistKV}, - StoreGateway: {API, Overrides, MemberlistKV}, - Purger: {Store, DeleteRequestsStore, API}, - All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler}, + API: {Server}, + Ring: {API, RuntimeConfig, MemberlistKV}, + Overrides: {RuntimeConfig}, + Distributor: {DistributorService, API}, + DistributorService: {Ring, Overrides}, + Store: {Overrides, DeleteRequestsStore}, + Ingester: {IngesterService, API}, + IngesterService: {Overrides, Store, RuntimeConfig, MemberlistKV}, + Flusher: {Store, API}, + Queryable: {Overrides, DistributorService, Store, Ring, API, StoreQueryable, MemberlistKV}, + Querier: {Queryable}, + StoreQueryable: {Overrides, Store, MemberlistKV}, + QueryFrontendTripperware: {API, Overrides, DeleteRequestsStore}, + QueryFrontend: {QueryFrontendTripperware}, + QueryScheduler: {API, Overrides}, + TableManager: {API}, + Ruler: {Overrides, DistributorService, Store, StoreQueryable, RulerStorage}, + Configs: {API}, + AlertManager: {API}, + Compactor: {API, MemberlistKV}, + StoreGateway: {API, Overrides, MemberlistKV}, + Purger: {Store, DeleteRequestsStore, API}, + All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler}, } for mod, targets := range deps { if err := mm.AddDependency(mod, targets...); err != nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/distributor/distributor.go b/vendor/github.com/cortexproject/cortex/pkg/distributor/distributor.go index 2b72301ab72c..8c32ddd9f008 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/distributor/distributor.go +++ b/vendor/github.com/cortexproject/cortex/pkg/distributor/distributor.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/scrape" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/instrument" @@ -446,6 +447,11 @@ func (d *Distributor) Push(ctx context.Context, req *client.WriteRequest) (*clie latestSampleTimestampMs = util.Max64(latestSampleTimestampMs, ts.Samples[len(ts.Samples)-1].TimestampMs) } + if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 { + l := relabel.Process(client.FromLabelAdaptersToLabels(ts.Labels), mrc...) + ts.Labels = client.FromLabelsToLabelAdapters(l) + } + // If we found both the cluster and replica labels, we only want to include the cluster label when // storing series in Cortex. If we kept the replica label we would end up with another series for the same // series we're trying to dedupe when HA tracking moves over to a different replica. @@ -637,14 +643,16 @@ func (d *Distributor) ForReplicationSet(ctx context.Context, replicationSet ring } // LabelValuesForLabelName returns all of the label values that are associated with a given label name. -func (d *Distributor) LabelValuesForLabelName(ctx context.Context, labelName model.LabelName) ([]string, error) { +func (d *Distributor) LabelValuesForLabelName(ctx context.Context, from, to model.Time, labelName model.LabelName) ([]string, error) { replicationSet, err := d.GetIngestersForMetadata(ctx) if err != nil { return nil, err } req := &client.LabelValuesRequest{ - LabelName: string(labelName), + LabelName: string(labelName), + StartTimestampMs: int64(from), + EndTimestampMs: int64(to), } resps, err := d.ForReplicationSet(ctx, replicationSet, func(ctx context.Context, client client.IngesterClient) (interface{}, error) { return client.LabelValues(ctx, req) @@ -668,13 +676,16 @@ func (d *Distributor) LabelValuesForLabelName(ctx context.Context, labelName mod } // LabelNames returns all of the label names. -func (d *Distributor) LabelNames(ctx context.Context) ([]string, error) { +func (d *Distributor) LabelNames(ctx context.Context, from, to model.Time) ([]string, error) { replicationSet, err := d.GetIngestersForMetadata(ctx) if err != nil { return nil, err } - req := &client.LabelNamesRequest{} + req := &client.LabelNamesRequest{ + StartTimestampMs: int64(from), + EndTimestampMs: int64(to), + } resps, err := d.ForReplicationSet(ctx, replicationSet, func(ctx context.Context, client client.IngesterClient) (interface{}, error) { return client.LabelNames(ctx, req) }) @@ -824,7 +835,7 @@ func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) { req := &client.UserStatsRequest{} ctx = user.InjectOrgID(ctx, "1") // fake: ingester insists on having an org ID // Not using d.ForReplicationSet(), so we can fail after first error. - replicationSet, err := d.ingestersRing.GetAll(ring.Read) + replicationSet, err := d.ingestersRing.GetAllHealthy(ring.Read) if err != nil { return nil, err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/distributor/query.go b/vendor/github.com/cortexproject/cortex/pkg/distributor/query.go index c6fd3aa68792..8f08268cd064 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/distributor/query.go +++ b/vendor/github.com/cortexproject/cortex/pkg/distributor/query.go @@ -88,7 +88,7 @@ func (d *Distributor) GetIngestersForQuery(ctx context.Context, matchers ...*lab lookbackPeriod := d.cfg.ShuffleShardingLookbackPeriod if shardSize > 0 && lookbackPeriod > 0 { - return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetAll(ring.Read) + return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetReplicationSetForOperation(ring.Read) } } @@ -101,7 +101,7 @@ func (d *Distributor) GetIngestersForQuery(ctx context.Context, matchers ...*lab } } - return d.ingestersRing.GetAll(ring.Read) + return d.ingestersRing.GetReplicationSetForOperation(ring.Read) } // GetIngestersForMetadata returns a replication set including all ingesters that should be queried @@ -119,11 +119,11 @@ func (d *Distributor) GetIngestersForMetadata(ctx context.Context) (ring.Replica lookbackPeriod := d.cfg.ShuffleShardingLookbackPeriod if shardSize > 0 && lookbackPeriod > 0 { - return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetAll(ring.Read) + return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetReplicationSetForOperation(ring.Read) } } - return d.ingestersRing.GetAll(ring.Read) + return d.ingestersRing.GetReplicationSetForOperation(ring.Read) } // queryIngesters queries the ingesters via the older, sample-based API. diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/config.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/config.go new file mode 100644 index 000000000000..d79aae8e6222 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/config.go @@ -0,0 +1,76 @@ +package frontend + +import ( + "flag" + "net/http" + + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + + "github.com/cortexproject/cortex/pkg/frontend/transport" + v1 "github.com/cortexproject/cortex/pkg/frontend/v1" + v2 "github.com/cortexproject/cortex/pkg/frontend/v2" + "github.com/cortexproject/cortex/pkg/util" +) + +// This struct combines several configuration options together to preserve backwards compatibility. +type CombinedFrontendConfig struct { + Handler transport.HandlerConfig `yaml:",inline"` + FrontendV1 v1.Config `yaml:",inline"` + FrontendV2 v2.Config `yaml:",inline"` + + CompressResponses bool `yaml:"compress_responses"` + DownstreamURL string `yaml:"downstream_url"` +} + +func (cfg *CombinedFrontendConfig) RegisterFlags(f *flag.FlagSet) { + cfg.Handler.RegisterFlags(f) + cfg.FrontendV1.RegisterFlags(f) + cfg.FrontendV2.RegisterFlags(f) + + f.BoolVar(&cfg.CompressResponses, "querier.compress-http-responses", false, "Compress HTTP responses.") + f.StringVar(&cfg.DownstreamURL, "frontend.downstream-url", "", "URL of downstream Prometheus.") +} + +// Initializes frontend (either V1 -- without scheduler, or V2 -- with scheduler) or no frontend at +// all if downstream Prometheus URL is used instead. +// +// Returned RoundTripper can be wrapped in more round-tripper middlewares, and then eventually registered +// into HTTP server using the Handler from this package. Returned RoundTripper is always non-nil +// (if there are no errors), and it uses the returned frontend (if any). +func InitFrontend(cfg CombinedFrontendConfig, limits v1.Limits, grpcListenPort int, log log.Logger, reg prometheus.Registerer) (http.RoundTripper, *v1.Frontend, *v2.Frontend, error) { + switch { + case cfg.DownstreamURL != "": + // If the user has specified a downstream Prometheus, then we should use that. + rt, err := NewDownstreamRoundTripper(cfg.DownstreamURL) + return rt, nil, nil, err + + case cfg.FrontendV2.SchedulerAddress != "": + // If query-scheduler address is configured, use Frontend. + if cfg.FrontendV2.Addr == "" { + addr, err := util.GetFirstAddressOf(cfg.FrontendV2.InfNames) + if err != nil { + return nil, nil, nil, errors.Wrap(err, "failed to get frontend address") + } + + cfg.FrontendV2.Addr = addr + } + + if cfg.FrontendV2.Port == 0 { + cfg.FrontendV2.Port = grpcListenPort + } + + fr, err := v2.NewFrontend(cfg.FrontendV2, log, reg) + return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), nil, fr, err + + default: + // No scheduler = use original frontend. + fr, err := v1.New(cfg.FrontendV1, limits, log, reg) + if err != nil { + return nil, nil, nil, err + } + + return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), fr, nil, err + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/downstream_roundtripper.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/downstream_roundtripper.go new file mode 100644 index 000000000000..f0c342eac4a8 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/downstream_roundtripper.go @@ -0,0 +1,40 @@ +package frontend + +import ( + "net/http" + "net/url" + "path" + + "github.com/opentracing/opentracing-go" +) + +// RoundTripper that forwards requests to downstream URL. +type downstreamRoundTripper struct { + downstreamURL *url.URL +} + +func NewDownstreamRoundTripper(downstreamURL string) (http.RoundTripper, error) { + u, err := url.Parse(downstreamURL) + if err != nil { + return nil, err + } + + return &downstreamRoundTripper{downstreamURL: u}, nil +} + +func (d downstreamRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(r.Context()) + if tracer != nil && span != nil { + carrier := opentracing.HTTPHeadersCarrier(r.Header) + err := tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier) + if err != nil { + return nil, err + } + } + + r.URL.Scheme = d.downstreamURL.Scheme + r.URL.Host = d.downstreamURL.Host + r.URL.Path = path.Join(d.downstreamURL.Path, r.URL.Path) + r.Host = "" + return http.DefaultTransport.RoundTrip(r) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/handler.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/handler.go new file mode 100644 index 000000000000..40f68ca78100 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/handler.go @@ -0,0 +1,135 @@ +package transport + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/httpgrpc/server" + + "github.com/cortexproject/cortex/pkg/util" +) + +const ( + // StatusClientClosedRequest is the status code for when a client request cancellation of an http request + StatusClientClosedRequest = 499 +) + +var ( + errCanceled = httpgrpc.Errorf(StatusClientClosedRequest, context.Canceled.Error()) + errDeadlineExceeded = httpgrpc.Errorf(http.StatusGatewayTimeout, context.DeadlineExceeded.Error()) + errRequestEntityTooLarge = httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "http: request body too large") +) + +// Config for a Handler. +type HandlerConfig struct { + LogQueriesLongerThan time.Duration `yaml:"log_queries_longer_than"` + MaxBodySize int64 `yaml:"max_body_size"` +} + +func (cfg *HandlerConfig) RegisterFlags(f *flag.FlagSet) { + f.DurationVar(&cfg.LogQueriesLongerThan, "frontend.log-queries-longer-than", 0, "Log queries that are slower than the specified duration. Set to 0 to disable. Set to < 0 to enable on all queries.") + f.Int64Var(&cfg.MaxBodySize, "frontend.max-body-size", 10*1024*1024, "Max body size for downstream prometheus.") +} + +// Handler accepts queries and forwards them to RoundTripper. It can log slow queries, +// but all other logic is inside the RoundTripper. +type Handler struct { + cfg HandlerConfig + log log.Logger + roundTripper http.RoundTripper +} + +// New creates a new frontend handler. +func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logger) http.Handler { + return &Handler{ + cfg: cfg, + log: log, + roundTripper: roundTripper, + } +} + +func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + _ = r.Body.Close() + }() + + // Buffer the body for later use to track slow queries. + var buf bytes.Buffer + r.Body = http.MaxBytesReader(w, r.Body, f.cfg.MaxBodySize) + r.Body = ioutil.NopCloser(io.TeeReader(r.Body, &buf)) + + startTime := time.Now() + resp, err := f.roundTripper.RoundTrip(r) + queryResponseTime := time.Since(startTime) + + if err != nil { + writeError(w, err) + return + } + + hs := w.Header() + for h, vs := range resp.Header { + hs[h] = vs + } + + w.WriteHeader(resp.StatusCode) + // we don't check for copy error as there is no much we can do at this point + _, _ = io.Copy(w, resp.Body) + + f.reportSlowQuery(queryResponseTime, r, buf) +} + +// reportSlowQuery reports slow queries if LogQueriesLongerThan is set to <0, where 0 disables logging +func (f *Handler) reportSlowQuery(queryResponseTime time.Duration, r *http.Request, bodyBuf bytes.Buffer) { + if f.cfg.LogQueriesLongerThan == 0 || queryResponseTime <= f.cfg.LogQueriesLongerThan { + return + } + + logMessage := []interface{}{ + "msg", "slow query detected", + "method", r.Method, + "host", r.Host, + "path", r.URL.Path, + "time_taken", queryResponseTime.String(), + } + + // use previously buffered body + r.Body = ioutil.NopCloser(&bodyBuf) + + // Ensure the form has been parsed so all the parameters are present + err := r.ParseForm() + if err != nil { + level.Warn(util.WithContext(r.Context(), f.log)).Log("msg", "unable to parse form for request", "err", err) + } + + // Attempt to iterate through the Form to log any filled in values + for k, v := range r.Form { + logMessage = append(logMessage, fmt.Sprintf("param_%s", k), strings.Join(v, ",")) + } + + level.Info(util.WithContext(r.Context(), f.log)).Log(logMessage...) +} + +func writeError(w http.ResponseWriter, err error) { + switch err { + case context.Canceled: + err = errCanceled + case context.DeadlineExceeded: + err = errDeadlineExceeded + default: + if strings.Contains(err.Error(), "http: request body too large") { + err = errRequestEntityTooLarge + } + } + server.WriteError(w, err) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/roundtripper.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/roundtripper.go new file mode 100644 index 000000000000..065d7fdca6e1 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/transport/roundtripper.go @@ -0,0 +1,47 @@ +package transport + +import ( + "bytes" + "context" + "io/ioutil" + "net/http" + + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/httpgrpc/server" +) + +// GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. +type GrpcRoundTripper interface { + RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) +} + +func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper { + return &grpcRoundTripperAdapter{roundTripper: r} +} + +// This adapter wraps GrpcRoundTripper and converted it into http.RoundTripper +type grpcRoundTripperAdapter struct { + roundTripper GrpcRoundTripper +} + +func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, error) { + req, err := server.HTTPRequest(r) + if err != nil { + return nil, err + } + + resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req) + if err != nil { + return nil, err + } + + httpResp := &http.Response{ + StatusCode: int(resp.Code), + Body: ioutil.NopCloser(bytes.NewReader(resp.Body)), + Header: http.Header{}, + } + for _, h := range resp.Headers { + httpResp.Header[h.Key] = h.Values + } + return httpResp, nil +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontend.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontend.go new file mode 100644 index 000000000000..44750d7fd386 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontend.go @@ -0,0 +1,282 @@ +package v1 + +import ( + "context" + "errors" + "flag" + "fmt" + "net/http" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/user" + + "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" + "github.com/cortexproject/cortex/pkg/scheduler/queue" + "github.com/cortexproject/cortex/pkg/util/grpcutil" +) + +var ( + errTooManyRequest = httpgrpc.Errorf(http.StatusTooManyRequests, "too many outstanding requests") +) + +// Config for a Frontend. +type Config struct { + MaxOutstandingPerTenant int `yaml:"max_outstanding_per_tenant"` +} + +// RegisterFlags adds the flags required to config this to the given FlagSet. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.IntVar(&cfg.MaxOutstandingPerTenant, "querier.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per frontend; requests beyond this error with HTTP 429.") +} + +type Limits interface { + // Returns max queriers to use per tenant, or 0 if shuffle sharding is disabled. + MaxQueriersPerUser(user string) int +} + +// Frontend queues HTTP requests, dispatches them to backends, and handles retries +// for requests which failed. +type Frontend struct { + cfg Config + log log.Logger + limits Limits + + requestQueue *queue.RequestQueue + + // Metrics. + numClients prometheus.GaugeFunc + queueDuration prometheus.Histogram +} + +type request struct { + enqueueTime time.Time + queueSpan opentracing.Span + originalCtx context.Context + + request *httpgrpc.HTTPRequest + err chan error + response chan *httpgrpc.HTTPResponse +} + +// New creates a new frontend. +func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer) (*Frontend, error) { + queueLength := promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_query_frontend_queue_length", + Help: "Number of queries in the queue.", + }, []string{"user"}) + + f := &Frontend{ + cfg: cfg, + log: log, + limits: limits, + requestQueue: queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, queueLength), + queueDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_query_frontend_queue_duration_seconds", + Help: "Time spend by requests queued.", + Buckets: prometheus.DefBuckets, + }), + } + + f.numClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_query_frontend_connected_clients", + Help: "Number of worker clients currently connected to the frontend.", + }, f.requestQueue.GetConnectedQuerierWorkersMetric) + + return f, nil +} + +// Close stops new requests and errors out any pending requests. +func (f *Frontend) Close() { + f.requestQueue.Stop() +} + +// RoundTripGRPC round trips a proto (instead of a HTTP request). +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { + // Propagate trace context in gRPC too - this will be ignored if using HTTP. + tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) + if tracer != nil && span != nil { + carrier := (*grpcutil.HttpgrpcHeadersCarrier)(req) + err := tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier) + if err != nil { + return nil, err + } + } + + request := request{ + request: req, + originalCtx: ctx, + + // Buffer of 1 to ensure response can be written by the server side + // of the Process stream, even if this goroutine goes away due to + // client context cancellation. + err: make(chan error, 1), + response: make(chan *httpgrpc.HTTPResponse, 1), + } + + if err := f.queueRequest(ctx, &request); err != nil { + return nil, err + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + + case resp := <-request.response: + return resp, nil + + case err := <-request.err: + return nil, err + } +} + +// Process allows backends to pull requests from the frontend. +func (f *Frontend) Process(server frontendv1pb.Frontend_ProcessServer) error { + querierID, err := getQuerierID(server) + if err != nil { + return err + } + + f.requestQueue.RegisterQuerierConnection(querierID) + defer f.requestQueue.UnregisterQuerierConnection(querierID) + + // If the downstream request(from querier -> frontend) is cancelled, + // we need to ping the condition variable to unblock getNextRequestForQuerier. + // Ideally we'd have ctx aware condition variables... + go func() { + <-server.Context().Done() + f.requestQueue.QuerierDisconnecting() + }() + + lastUserIndex := queue.FirstUser() + + for { + reqWrapper, idx, err := f.requestQueue.GetNextRequestForQuerier(server.Context(), lastUserIndex, querierID) + if err != nil { + return err + } + lastUserIndex = idx + + req := reqWrapper.(*request) + + f.queueDuration.Observe(time.Since(req.enqueueTime).Seconds()) + req.queueSpan.Finish() + + /* + We want to dequeue the next unexpired request from the chosen tenant queue. + The chance of choosing a particular tenant for dequeueing is (1/active_tenants). + This is problematic under load, especially with other middleware enabled such as + querier.split-by-interval, where one request may fan out into many. + If expired requests aren't exhausted before checking another tenant, it would take + n_active_tenants * n_expired_requests_at_front_of_queue requests being processed + before an active request was handled for the tenant in question. + If this tenant meanwhile continued to queue requests, + it's possible that it's own queue would perpetually contain only expired requests. + */ + if req.originalCtx.Err() != nil { + lastUserIndex = lastUserIndex.ReuseLastUser() + continue + } + + // Handle the stream sending & receiving on a goroutine so we can + // monitoring the contexts in a select and cancel things appropriately. + resps := make(chan *httpgrpc.HTTPResponse, 1) + errs := make(chan error, 1) + go func() { + err = server.Send(&frontendv1pb.FrontendToClient{ + Type: frontendv1pb.HTTP_REQUEST, + HttpRequest: req.request, + }) + if err != nil { + errs <- err + return + } + + resp, err := server.Recv() + if err != nil { + errs <- err + return + } + + resps <- resp.HttpResponse + }() + + select { + // If the upstream request is cancelled, we need to cancel the + // downstream req. Only way we can do that is to close the stream. + // The worker client is expecting this semantics. + case <-req.originalCtx.Done(): + return req.originalCtx.Err() + + // Is there was an error handling this request due to network IO, + // then error out this upstream request _and_ stream. + case err := <-errs: + req.err <- err + return err + + // Happy path: propagate the response. + case resp := <-resps: + req.response <- resp + } + } +} + +func getQuerierID(server frontendv1pb.Frontend_ProcessServer) (string, error) { + err := server.Send(&frontendv1pb.FrontendToClient{ + Type: frontendv1pb.GET_ID, + // Old queriers don't support GET_ID, and will try to use the request. + // To avoid confusing them, include dummy request. + HttpRequest: &httpgrpc.HTTPRequest{ + Method: "GET", + Url: "/invalid_request_sent_by_frontend", + }, + }) + + if err != nil { + return "", err + } + + resp, err := server.Recv() + + // Old queriers will return empty string, which is fine. All old queriers will be + // treated as single querier with lot of connections. + // (Note: if resp is nil, GetClientID() returns "") + return resp.GetClientID(), err +} + +func (f *Frontend) queueRequest(ctx context.Context, req *request) error { + userID, err := user.ExtractOrgID(ctx) + if err != nil { + return err + } + + req.enqueueTime = time.Now() + req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") + + maxQueriers := f.limits.MaxQueriersPerUser(userID) + + err = f.requestQueue.EnqueueRequest(userID, req, maxQueriers, nil) + if err == queue.ErrTooManyRequests { + return errTooManyRequest + } + return err +} + +// CheckReady determines if the query frontend is ready. Function parameters/return +// chosen to match the same method in the ingester +func (f *Frontend) CheckReady(_ context.Context) error { + // if we have more than one querier connected we will consider ourselves ready + connectedClients := f.requestQueue.GetConnectedQuerierWorkersMetric() + if connectedClients > 0 { + return nil + } + + msg := fmt.Sprintf("not ready: number of queriers connected to query-frontend is %d", int64(connectedClients)) + level.Info(f.log).Log("msg", msg) + return errors.New(msg) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.pb.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.pb.go similarity index 89% rename from vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.pb.go rename to vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.pb.go index 841fc7148cb6..0729cc1df151 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.pb.go +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.pb.go @@ -1,7 +1,10 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: frontend.proto -package frontend +// Protobuf package should not be changed when moving around go packages +// in order to not break backward compatibility. + +package frontendv1pb import ( context "context" @@ -163,30 +166,30 @@ func init() { func init() { proto.RegisterFile("frontend.proto", fileDescriptor_eca3873955a29cfe) } var fileDescriptor_eca3873955a29cfe = []byte{ - // 362 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x91, 0xb1, 0x4e, 0x2a, 0x41, - 0x14, 0x86, 0x67, 0x6e, 0x08, 0x97, 0x3b, 0x10, 0xb2, 0x99, 0xe4, 0xde, 0x90, 0x2d, 0x26, 0x64, - 0x73, 0x0b, 0x62, 0xe2, 0xae, 0x41, 0x13, 0x13, 0x0b, 0x0b, 0x05, 0x91, 0x0e, 0x97, 0xb1, 0xb1, - 0x21, 0xb2, 0x0e, 0x0b, 0x2a, 0x7b, 0xd6, 0xdd, 0x41, 0x42, 0xe7, 0x23, 0xf8, 0x18, 0x3e, 0x8a, - 0x25, 0x25, 0xa5, 0x0c, 0x8d, 0x25, 0x8f, 0x60, 0x98, 0x85, 0x15, 0xe9, 0xce, 0x9f, 0xff, 0x3f, - 0xe7, 0x3b, 0x33, 0x87, 0x14, 0x7b, 0x11, 0x04, 0x52, 0x04, 0x77, 0x76, 0x18, 0x81, 0x04, 0x9a, - 0xdb, 0x68, 0x73, 0xdf, 0x1f, 0xc8, 0xfe, 0xa8, 0x6b, 0x7b, 0x30, 0x74, 0x7c, 0xf0, 0xc1, 0xd1, - 0x81, 0xee, 0xa8, 0xa7, 0x95, 0x16, 0xba, 0x4a, 0x1a, 0xcd, 0xa3, 0xad, 0xf8, 0x58, 0xdc, 0x3e, - 0x8b, 0x31, 0x44, 0x0f, 0xb1, 0xe3, 0xc1, 0x70, 0x08, 0x81, 0xd3, 0x97, 0x32, 0xf4, 0xa3, 0xd0, - 0x4b, 0x8b, 0xa4, 0xcb, 0x02, 0x62, 0x5c, 0xac, 0x81, 0x1c, 0xce, 0x1f, 0x07, 0x22, 0x90, 0xf4, - 0x98, 0xe4, 0x57, 0x29, 0x57, 0x3c, 0x8d, 0x44, 0x2c, 0x4b, 0xb8, 0x8c, 0x2b, 0xf9, 0xea, 0x5f, - 0x3b, 0xed, 0xbc, 0xe4, 0xbc, 0xb5, 0x36, 0xdd, 0xed, 0x24, 0xb5, 0x48, 0x46, 0x4e, 0x42, 0x51, - 0xfa, 0x55, 0xc6, 0x95, 0x62, 0xb5, 0x68, 0xa7, 0x4f, 0xe3, 0x93, 0x50, 0xb8, 0xda, 0xb3, 0xee, - 0x89, 0x91, 0x60, 0x38, 0x6c, 0xc0, 0xf4, 0x84, 0x14, 0x92, 0x31, 0x71, 0x08, 0x41, 0x2c, 0xd6, - 0xc4, 0x7f, 0xbb, 0xc4, 0xc4, 0x75, 0x7f, 0x64, 0xa9, 0x49, 0x72, 0x9e, 0x9e, 0xd7, 0xac, 0x69, - 0xee, 0x1f, 0x37, 0xd5, 0x7b, 0xff, 0x49, 0x66, 0x45, 0xa6, 0x06, 0x29, 0xac, 0x26, 0x74, 0xdc, - 0xfa, 0xd5, 0x75, 0xbd, 0xcd, 0x0d, 0x44, 0x09, 0xc9, 0x36, 0xea, 0xbc, 0xd3, 0xac, 0x19, 0xb8, - 0xda, 0x26, 0xb9, 0x74, 0x93, 0x06, 0xf9, 0xdd, 0x8a, 0xc0, 0x13, 0x71, 0x4c, 0xcd, 0xef, 0xf5, - 0x77, 0x17, 0x36, 0xb7, 0xbc, 0xdd, 0xdf, 0xb3, 0x50, 0x05, 0x1f, 0xe0, 0xb3, 0xd3, 0xe9, 0x9c, - 0xa1, 0xd9, 0x9c, 0xa1, 0xe5, 0x9c, 0xe1, 0x17, 0xc5, 0xf0, 0x9b, 0x62, 0xf8, 0x5d, 0x31, 0x3c, - 0x55, 0x0c, 0x7f, 0x28, 0x86, 0x3f, 0x15, 0x43, 0x4b, 0xc5, 0xf0, 0xeb, 0x82, 0xa1, 0xe9, 0x82, - 0xa1, 0xd9, 0x82, 0xa1, 0x9b, 0xf4, 0xf8, 0xdd, 0xac, 0x3e, 0xcf, 0xe1, 0x57, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x14, 0x4d, 0x19, 0x1d, 0x1f, 0x02, 0x00, 0x00, + // 364 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x91, 0xc1, 0x4e, 0xea, 0x40, + 0x14, 0x86, 0x67, 0x6e, 0x08, 0x97, 0x3b, 0x10, 0xd2, 0x4c, 0x72, 0x6f, 0x48, 0x17, 0x13, 0xd2, + 0xdc, 0x05, 0x31, 0xb1, 0x55, 0x34, 0x31, 0x71, 0x89, 0x20, 0xb2, 0xc3, 0x52, 0x37, 0x6e, 0x88, + 0xad, 0x43, 0x41, 0xa5, 0xa7, 0xb6, 0x03, 0x84, 0x9d, 0x8f, 0xe0, 0x63, 0xf8, 0x28, 0x2e, 0x59, + 0xb2, 0x94, 0x61, 0xe3, 0x92, 0x47, 0x30, 0x4c, 0xa1, 0x22, 0xbb, 0xf3, 0xe7, 0xff, 0xcf, 0xff, + 0x9d, 0x76, 0x48, 0xb1, 0x17, 0x41, 0x20, 0x78, 0x70, 0x6f, 0x86, 0x11, 0x08, 0xa0, 0xb9, 0xad, + 0xd6, 0x0f, 0xfd, 0x81, 0xe8, 0x8f, 0x5c, 0xd3, 0x83, 0xa1, 0xe5, 0x83, 0x0f, 0x96, 0x0a, 0xb8, + 0xa3, 0x9e, 0x52, 0x4a, 0xa8, 0x29, 0x59, 0xd4, 0x4f, 0x77, 0xe2, 0x13, 0x7e, 0x37, 0xe6, 0x13, + 0x88, 0x1e, 0x63, 0xcb, 0x83, 0xe1, 0x10, 0x02, 0xab, 0x2f, 0x44, 0xe8, 0x47, 0xa1, 0x97, 0x0e, + 0xc9, 0x96, 0x01, 0x44, 0xbb, 0xdc, 0x00, 0x1d, 0xb8, 0x78, 0x1a, 0xf0, 0x40, 0xd0, 0x33, 0x92, + 0x5f, 0xa7, 0x6c, 0xfe, 0x3c, 0xe2, 0xb1, 0x28, 0xe1, 0x32, 0xae, 0xe4, 0xab, 0x7f, 0xcd, 0x74, + 0xf3, 0xca, 0x71, 0xda, 0x1b, 0xd3, 0xde, 0x4d, 0x52, 0x83, 0x64, 0xc4, 0x34, 0xe4, 0xa5, 0x5f, + 0x65, 0x5c, 0x29, 0x56, 0x8b, 0x66, 0xfa, 0x69, 0xce, 0x34, 0xe4, 0xb6, 0xf2, 0x8c, 0x07, 0xa2, + 0x25, 0x18, 0x07, 0xb6, 0x60, 0x7a, 0x4e, 0x0a, 0x49, 0x4d, 0x1c, 0x42, 0x10, 0xf3, 0x0d, 0xf1, + 0xdf, 0x3e, 0x31, 0x71, 0xed, 0x1f, 0x59, 0xaa, 0x93, 0x9c, 0xa7, 0xfa, 0x5a, 0x75, 0xc5, 0xfd, + 0x63, 0xa7, 0xfa, 0xe0, 0x3f, 0xc9, 0xac, 0xc9, 0x54, 0x23, 0x85, 0x75, 0x43, 0xd7, 0x6e, 0x5c, + 0xdf, 0x34, 0x3a, 0x8e, 0x86, 0x28, 0x21, 0xd9, 0x66, 0xc3, 0xe9, 0xb6, 0xea, 0x1a, 0xae, 0x76, + 0x48, 0x2e, 0xbd, 0xa4, 0x49, 0x7e, 0xb7, 0x23, 0xf0, 0x78, 0x1c, 0x53, 0xfd, 0xfb, 0xfc, 0xfd, + 0x83, 0xf5, 0x1d, 0x6f, 0xff, 0xef, 0x19, 0xa8, 0x82, 0x8f, 0x70, 0xad, 0x36, 0x5b, 0x30, 0x34, + 0x5f, 0x30, 0xb4, 0x5a, 0x30, 0xfc, 0x22, 0x19, 0x7e, 0x93, 0x0c, 0xbf, 0x4b, 0x86, 0x67, 0x92, + 0xe1, 0x0f, 0xc9, 0xf0, 0xa7, 0x64, 0x68, 0x25, 0x19, 0x7e, 0x5d, 0x32, 0x34, 0x5b, 0x32, 0x34, + 0x5f, 0x32, 0x74, 0x5b, 0xd8, 0xd6, 0x8e, 0x8f, 0x43, 0xd7, 0xcd, 0xaa, 0x27, 0x3a, 0xf9, 0x0a, + 0x00, 0x00, 0xff, 0xff, 0x85, 0xf3, 0xb6, 0x2a, 0x23, 0x02, 0x00, 0x00, } func (x Type) String() string { @@ -255,7 +258,7 @@ func (this *FrontendToClient) GoString() string { return "nil" } s := make([]string, 0, 6) - s = append(s, "&frontend.FrontendToClient{") + s = append(s, "&frontendv1pb.FrontendToClient{") if this.HttpRequest != nil { s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") } @@ -268,7 +271,7 @@ func (this *ClientToFrontend) GoString() string { return "nil" } s := make([]string, 0, 6) - s = append(s, "&frontend.ClientToFrontend{") + s = append(s, "&frontendv1pb.ClientToFrontend{") if this.HttpResponse != nil { s = append(s, "HttpResponse: "+fmt.Sprintf("%#v", this.HttpResponse)+",\n") } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.proto b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.proto similarity index 82% rename from vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.proto rename to vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.proto index 077c1db0d620..1d63ce2b950c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.proto +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb/frontend.proto @@ -1,8 +1,10 @@ syntax = "proto3"; +// Protobuf package should not be changed when moving around go packages +// in order to not break backward compatibility. package frontend; -option go_package = "frontend"; +option go_package = "frontendv1pb"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "github.com/weaveworks/common/httpgrpc/httpgrpc.proto"; diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend.go new file mode 100644 index 000000000000..2dd270641361 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend.go @@ -0,0 +1,309 @@ +package v2 + +import ( + "context" + "flag" + "fmt" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/user" + "go.uber.org/atomic" + + "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/grpcutil" + "github.com/cortexproject/cortex/pkg/util/services" +) + +// Config for a Frontend. +type Config struct { + SchedulerAddress string `yaml:"scheduler_address"` + DNSLookupPeriod time.Duration `yaml:"scheduler_dns_lookup_period"` + WorkerConcurrency int `yaml:"scheduler_worker_concurrency"` + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` + + // Used to find local IP address, that is sent to scheduler and querier-worker. + InfNames []string `yaml:"instance_interface_names"` + + // If set, address is not computed from interfaces. + Addr string `yaml:"address" doc:"hidden"` + Port int `doc:"hidden"` +} + +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&cfg.SchedulerAddress, "frontend.scheduler-address", "", "DNS hostname used for finding query-schedulers.") + f.DurationVar(&cfg.DNSLookupPeriod, "frontend.scheduler-dns-lookup-period", 10*time.Second, "How often to resolve the scheduler-address, in order to look for new query-scheduler instances.") + f.IntVar(&cfg.WorkerConcurrency, "frontend.scheduler-worker-concurrency", 5, "Number of concurrent workers forwarding queries to single query-scheduler.") + + cfg.InfNames = []string{"eth0", "en0"} + f.Var((*flagext.StringSlice)(&cfg.InfNames), "frontend.instance-interface-names", "Name of network interface to read address from. This address is sent to query-scheduler and querier, which uses it to send the query response back to query-frontend.") + f.StringVar(&cfg.Addr, "frontend.instance-addr", "", "IP address to advertise to querier (via scheduler) (resolved via interfaces by default).") + f.IntVar(&cfg.Port, "frontend.instance-port", 0, "Port to advertise to querier (via scheduler) (defaults to server.grpc-listen-port).") + + cfg.GRPCClientConfig.RegisterFlagsWithPrefix("frontend.grpc-client-config", f) +} + +// Frontend implements GrpcRoundTripper. It queues HTTP requests, +// dispatches them to backends via gRPC, and handles retries for requests which failed. +type Frontend struct { + services.Service + + cfg Config + log log.Logger + + lastQueryID atomic.Uint64 + + // frontend workers will read from this channel, and send request to scheduler. + requestsCh chan *frontendRequest + + schedulerWorkers *frontendSchedulerWorkers + requests *requestsInProgress +} + +type frontendRequest struct { + queryID uint64 + request *httpgrpc.HTTPRequest + userID string + + cancel context.CancelFunc + + enqueue chan enqueueResult + response chan *httpgrpc.HTTPResponse +} + +type enqueueStatus int + +const ( + // Sent to scheduler successfully, and frontend should wait for response now. + waitForResponse enqueueStatus = iota + + // Failed to forward request to scheduler, frontend will try again. + failed +) + +type enqueueResult struct { + status enqueueStatus + + cancelCh chan<- uint64 // Channel that can be used for request cancellation. If nil, cancellation is not possible. +} + +// New creates a new frontend. +func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer) (*Frontend, error) { + requestsCh := make(chan *frontendRequest) + + schedulerWorkers, err := newFrontendSchedulerWorkers(cfg, fmt.Sprintf("%s:%d", cfg.Addr, cfg.Port), requestsCh, log) + if err != nil { + return nil, err + } + + f := &Frontend{ + cfg: cfg, + log: log, + requestsCh: requestsCh, + schedulerWorkers: schedulerWorkers, + requests: newRequestsInProgress(), + } + // Randomize to avoid getting responses from queries sent before restart, which could lead to mixing results + // between different queries. Note that frontend verifies the user, so it cannot leak results between tenants. + // This isn't perfect, but better than nothing. + f.lastQueryID.Store(rand.Uint64()) + + promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_query_frontend_queries_in_progress", + Help: "Number of queries in progress handled by this frontend.", + }, func() float64 { + return float64(f.requests.count()) + }) + + promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_query_frontend_connected_schedulers", + Help: "Number of schedulers this frontend is connected to.", + }, func() float64 { + return float64(f.schedulerWorkers.getWorkersCount()) + }) + + f.Service = services.NewIdleService(f.starting, f.stopping) + return f, nil +} + +func (f *Frontend) starting(ctx context.Context) error { + return errors.Wrap(services.StartAndAwaitRunning(ctx, f.schedulerWorkers), "failed to start frontend scheduler workers") +} + +func (f *Frontend) stopping(_ error) error { + return errors.Wrap(services.StopAndAwaitTerminated(context.Background(), f.schedulerWorkers), "failed to stop frontend scheduler workers") +} + +// RoundTripGRPC round trips a proto (instead of a HTTP request). +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { + if s := f.State(); s != services.Running { + return nil, fmt.Errorf("frontend not running: %v", s) + } + + userID, err := user.ExtractOrgID(ctx) + if err != nil { + return nil, err + } + + // Propagate trace context in gRPC too - this will be ignored if using HTTP. + tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) + if tracer != nil && span != nil { + carrier := (*grpcutil.HttpgrpcHeadersCarrier)(req) + if err := tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier); err != nil { + return nil, err + } + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + freq := &frontendRequest{ + queryID: f.lastQueryID.Inc(), + request: req, + userID: userID, + + cancel: cancel, + + // Buffer of 1 to ensure response or error can be written to the channel + // even if this goroutine goes away due to client context cancellation. + enqueue: make(chan enqueueResult, 1), + response: make(chan *httpgrpc.HTTPResponse, 1), + } + + f.requests.put(freq) + defer f.requests.delete(freq.queryID) + + retries := f.cfg.WorkerConcurrency + 1 // To make sure we hit at least two different schedulers. + +enqueueAgain: + select { + case <-ctx.Done(): + return nil, ctx.Err() + + case f.requestsCh <- freq: + // Enqueued, let's wait for response. + } + + var cancelCh chan<- uint64 + + select { + case <-ctx.Done(): + return nil, ctx.Err() + + case enqRes := <-freq.enqueue: + if enqRes.status == waitForResponse { + cancelCh = enqRes.cancelCh + break // go wait for response. + } else if enqRes.status == failed { + retries-- + if retries > 0 { + goto enqueueAgain + } + } + + return nil, httpgrpc.Errorf(http.StatusInternalServerError, "failed to enqueue request") + } + + select { + case <-ctx.Done(): + if cancelCh != nil { + select { + case cancelCh <- freq.queryID: + // cancellation sent. + default: + // failed to cancel, ignore. + } + } + return nil, ctx.Err() + + case resp := <-freq.response: + return resp, nil + } +} + +func (f *Frontend) QueryResult(ctx context.Context, qrReq *frontendv2pb.QueryResultRequest) (*frontendv2pb.QueryResultResponse, error) { + userID, err := user.ExtractOrgID(ctx) + if err != nil { + return nil, err + } + + req := f.requests.get(qrReq.QueryID) + // It is possible that some old response belonging to different user was received, if frontend has restarted. + // To avoid leaking query results between users, we verify the user here. + // To avoid mixing results from different queries, we randomize queryID counter on start. + if req != nil && req.userID == userID { + select { + case req.response <- qrReq.HttpResponse: + // Should always be possible, unless QueryResult is called multiple times with the same queryID. + default: + level.Warn(f.log).Log("msg", "failed to write query result to the response channel", "queryID", qrReq.QueryID, "user", userID) + } + } + + return &frontendv2pb.QueryResultResponse{}, nil +} + +// CheckReady determines if the query frontend is ready. Function parameters/return +// chosen to match the same method in the ingester +func (f *Frontend) CheckReady(_ context.Context) error { + workers := f.schedulerWorkers.getWorkersCount() + + // If frontend is connected to at least one scheduler, we are ready. + if workers > 0 { + return nil + } + + msg := fmt.Sprintf("not ready: number of schedulers this worker is connected to is %d", workers) + level.Info(f.log).Log("msg", msg) + return errors.New(msg) +} + +type requestsInProgress struct { + mu sync.Mutex + requests map[uint64]*frontendRequest +} + +func newRequestsInProgress() *requestsInProgress { + return &requestsInProgress{ + requests: map[uint64]*frontendRequest{}, + } +} + +func (r *requestsInProgress) count() int { + r.mu.Lock() + defer r.mu.Unlock() + + return len(r.requests) +} + +func (r *requestsInProgress) put(req *frontendRequest) { + r.mu.Lock() + defer r.mu.Unlock() + + r.requests[req.queryID] = req +} + +func (r *requestsInProgress) delete(queryID uint64) { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.requests, queryID) +} + +func (r *requestsInProgress) get(queryID uint64) *frontendRequest { + r.mu.Lock() + defer r.mu.Unlock() + + return r.requests[queryID] +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend_scheduler_worker.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend_scheduler_worker.go new file mode 100644 index 000000000000..9500669ce8af --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontend_scheduler_worker.go @@ -0,0 +1,310 @@ +package v2 + +import ( + "context" + "net/http" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/weaveworks/common/httpgrpc" + "google.golang.org/grpc" + + "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/services" +) + +type frontendSchedulerWorkers struct { + services.Service + + cfg Config + log log.Logger + frontendAddress string + + // Channel with requests that should be forwarded to the scheduler. + requestsCh <-chan *frontendRequest + + watcher services.Service + + mu sync.Mutex + // Set to nil when stop is called... no more workers are created afterwards. + workers map[string]*frontendSchedulerWorker +} + +func newFrontendSchedulerWorkers(cfg Config, frontendAddress string, requestsCh <-chan *frontendRequest, log log.Logger) (*frontendSchedulerWorkers, error) { + f := &frontendSchedulerWorkers{ + cfg: cfg, + log: log, + frontendAddress: frontendAddress, + requestsCh: requestsCh, + workers: map[string]*frontendSchedulerWorker{}, + } + + w, err := util.NewDNSWatcher(cfg.SchedulerAddress, cfg.DNSLookupPeriod, f) + if err != nil { + return nil, err + } + + f.watcher = w + f.Service = services.NewIdleService(f.starting, f.stopping) + return f, nil +} + +func (f *frontendSchedulerWorkers) starting(ctx context.Context) error { + return services.StartAndAwaitRunning(ctx, f.watcher) +} + +func (f *frontendSchedulerWorkers) stopping(_ error) error { + err := services.StopAndAwaitTerminated(context.Background(), f.watcher) + + f.mu.Lock() + defer f.mu.Unlock() + + for _, w := range f.workers { + w.stop() + } + f.workers = nil + + return err +} + +func (f *frontendSchedulerWorkers) AddressAdded(address string) { + f.mu.Lock() + ws := f.workers + w := f.workers[address] + f.mu.Unlock() + + // Already stopped or we already have worker for this address. + if ws == nil || w != nil { + return + } + + level.Info(f.log).Log("msg", "adding connection to scheduler", "addr", address) + conn, err := f.connectToScheduler(context.Background(), address) + if err != nil { + level.Error(f.log).Log("msg", "error connecting to scheduler", "addr", address, "err", err) + return + } + + // No worker for this address yet, start a new one. + w = newFrontendSchedulerWorker(conn, address, f.frontendAddress, f.requestsCh, f.cfg.WorkerConcurrency, f.log) + + f.mu.Lock() + defer f.mu.Unlock() + + // Can be nil if stopping has been called already. + if f.workers != nil { + f.workers[address] = w + w.start() + } +} + +func (f *frontendSchedulerWorkers) AddressRemoved(address string) { + level.Info(f.log).Log("msg", "removing connection to scheduler", "addr", address) + + f.mu.Lock() + // This works fine if f.workers is nil already. + w := f.workers[address] + delete(f.workers, address) + f.mu.Unlock() + + if w != nil { + w.stop() + } +} + +// Get number of workers. +func (f *frontendSchedulerWorkers) getWorkersCount() int { + f.mu.Lock() + defer f.mu.Unlock() + + return len(f.workers) +} + +func (f *frontendSchedulerWorkers) connectToScheduler(ctx context.Context, address string) (*grpc.ClientConn, error) { + // Because we only use single long-running method, it doesn't make sense to inject user ID, send over tracing or add metrics. + opts, err := f.cfg.GRPCClientConfig.DialOption(nil, nil) + if err != nil { + return nil, err + } + + conn, err := grpc.DialContext(ctx, address, opts...) + if err != nil { + return nil, err + } + return conn, nil +} + +// Worker managing single gRPC connection to Scheduler. Each worker starts multiple goroutines for forwarding +// requests and cancellations to scheduler. +type frontendSchedulerWorker struct { + log log.Logger + + conn *grpc.ClientConn + concurrency int + schedulerAddr string + frontendAddr string + + // Context and cancellation used by individual goroutines. + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + // Shared between all frontend workers. + requestCh <-chan *frontendRequest + + // Cancellation requests for this scheduler are received via this channel. It is passed to frontend after + // query has been enqueued to scheduler. + cancelCh chan uint64 +} + +func newFrontendSchedulerWorker(conn *grpc.ClientConn, schedulerAddr string, frontendAddr string, requestCh <-chan *frontendRequest, concurrency int, log log.Logger) *frontendSchedulerWorker { + w := &frontendSchedulerWorker{ + log: log, + conn: conn, + concurrency: concurrency, + schedulerAddr: schedulerAddr, + frontendAddr: frontendAddr, + requestCh: requestCh, + cancelCh: make(chan uint64), + } + w.ctx, w.cancel = context.WithCancel(context.Background()) + + return w +} + +func (w *frontendSchedulerWorker) start() { + client := schedulerpb.NewSchedulerForFrontendClient(w.conn) + for i := 0; i < w.concurrency; i++ { + w.wg.Add(1) + go func() { + defer w.wg.Done() + w.runOne(w.ctx, client) + }() + } +} + +func (w *frontendSchedulerWorker) stop() { + w.cancel() + w.wg.Wait() + if err := w.conn.Close(); err != nil { + level.Error(w.log).Log("msg", "error while closing connection to scheduler", "err", err) + } +} + +func (w *frontendSchedulerWorker) runOne(ctx context.Context, client schedulerpb.SchedulerForFrontendClient) { + backoffConfig := util.BackoffConfig{ + MinBackoff: 50 * time.Millisecond, + MaxBackoff: 1 * time.Second, + } + + backoff := util.NewBackoff(ctx, backoffConfig) + for backoff.Ongoing() { + loop, loopErr := client.FrontendLoop(ctx) + if loopErr != nil { + level.Error(w.log).Log("msg", "error contacting scheduler", "err", loopErr, "addr", w.schedulerAddr) + backoff.Wait() + continue + } + + loopErr = w.schedulerLoop(ctx, loop) + if closeErr := loop.CloseSend(); closeErr != nil { + level.Debug(w.log).Log("msg", "failed to close frontend loop", "err", loopErr, "addr", w.schedulerAddr) + } + + if loopErr != nil { + level.Error(w.log).Log("msg", "error sending requests to scheduler", "err", loopErr, "addr", w.schedulerAddr) + backoff.Wait() + continue + } + + backoff.Reset() + } +} + +func (w *frontendSchedulerWorker) schedulerLoop(ctx context.Context, loop schedulerpb.SchedulerForFrontend_FrontendLoopClient) error { + if err := loop.Send(&schedulerpb.FrontendToScheduler{ + Type: schedulerpb.INIT, + FrontendAddress: w.frontendAddr, + }); err != nil { + return err + } + + if resp, err := loop.Recv(); err != nil || resp.Status != schedulerpb.OK { + if err != nil { + return err + } + return errors.Errorf("unexpected status received: %v", resp.Status) + } + + for { + select { + case <-ctx.Done(): + return nil + + case req := <-w.requestCh: + err := loop.Send(&schedulerpb.FrontendToScheduler{ + Type: schedulerpb.ENQUEUE, + QueryID: req.queryID, + UserID: req.userID, + HttpRequest: req.request, + FrontendAddress: w.frontendAddr, + }) + + if err != nil { + req.enqueue <- enqueueResult{status: failed} + return err + } + + resp, err := loop.Recv() + if err != nil { + req.enqueue <- enqueueResult{status: failed} + return err + } + + switch resp.Status { + case schedulerpb.OK: + req.enqueue <- enqueueResult{status: waitForResponse, cancelCh: w.cancelCh} + // Response will come from querier. + + case schedulerpb.SHUTTING_DOWN: + // Scheduler is shutting down, report failure to enqueue and stop this loop. + req.enqueue <- enqueueResult{status: failed} + return errors.New("scheduler is shutting down") + + case schedulerpb.ERROR: + req.enqueue <- enqueueResult{status: waitForResponse} + req.response <- &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + + case schedulerpb.TOO_MANY_REQUESTS_PER_TENANT: + req.enqueue <- enqueueResult{status: waitForResponse} + req.response <- &httpgrpc.HTTPResponse{ + Code: http.StatusTooManyRequests, + Body: []byte("too many outstanding requests"), + } + } + + case reqID := <-w.cancelCh: + err := loop.Send(&schedulerpb.FrontendToScheduler{ + Type: schedulerpb.CANCEL, + QueryID: reqID, + }) + + if err != nil { + return err + } + + // Not interested in cancellation response. + _, err = loop.Recv() + if err != nil { + return err + } + } + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.pb.go b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.pb.go new file mode 100644 index 000000000000..940134933f02 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.pb.go @@ -0,0 +1,711 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: frontend.proto + +package frontendv2pb + +import ( + context "context" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + httpgrpc "github.com/weaveworks/common/httpgrpc" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type QueryResultRequest struct { + QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` + HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,2,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` +} + +func (m *QueryResultRequest) Reset() { *m = QueryResultRequest{} } +func (*QueryResultRequest) ProtoMessage() {} +func (*QueryResultRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_eca3873955a29cfe, []int{0} +} +func (m *QueryResultRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *QueryResultRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_QueryResultRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *QueryResultRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResultRequest.Merge(m, src) +} +func (m *QueryResultRequest) XXX_Size() int { + return m.Size() +} +func (m *QueryResultRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryResultRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryResultRequest proto.InternalMessageInfo + +func (m *QueryResultRequest) GetQueryID() uint64 { + if m != nil { + return m.QueryID + } + return 0 +} + +func (m *QueryResultRequest) GetHttpResponse() *httpgrpc.HTTPResponse { + if m != nil { + return m.HttpResponse + } + return nil +} + +type QueryResultResponse struct { +} + +func (m *QueryResultResponse) Reset() { *m = QueryResultResponse{} } +func (*QueryResultResponse) ProtoMessage() {} +func (*QueryResultResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_eca3873955a29cfe, []int{1} +} +func (m *QueryResultResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *QueryResultResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_QueryResultResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *QueryResultResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResultResponse.Merge(m, src) +} +func (m *QueryResultResponse) XXX_Size() int { + return m.Size() +} +func (m *QueryResultResponse) XXX_DiscardUnknown() { + xxx_messageInfo_QueryResultResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryResultResponse proto.InternalMessageInfo + +func init() { + proto.RegisterType((*QueryResultRequest)(nil), "frontendv2pb.QueryResultRequest") + proto.RegisterType((*QueryResultResponse)(nil), "frontendv2pb.QueryResultResponse") +} + +func init() { proto.RegisterFile("frontend.proto", fileDescriptor_eca3873955a29cfe) } + +var fileDescriptor_eca3873955a29cfe = []byte{ + // 298 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x2b, 0xca, 0xcf, + 0x2b, 0x49, 0xcd, 0x4b, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x81, 0xf1, 0xcb, 0x8c, + 0x0a, 0x92, 0xa4, 0x74, 0xd3, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0xd3, + 0xf3, 0xd3, 0xf3, 0xf5, 0xc1, 0x8a, 0x92, 0x4a, 0xd3, 0xc0, 0x3c, 0x30, 0x07, 0xcc, 0x82, 0x68, + 0x96, 0x32, 0x41, 0x52, 0x5e, 0x9e, 0x9a, 0x58, 0x96, 0x5a, 0x9e, 0x5f, 0x94, 0x5d, 0xac, 0x9f, + 0x9c, 0x9f, 0x9b, 0x9b, 0x9f, 0xa7, 0x9f, 0x51, 0x52, 0x52, 0x90, 0x5e, 0x54, 0x90, 0x0c, 0x67, + 0x40, 0x74, 0x29, 0x65, 0x71, 0x09, 0x05, 0x96, 0xa6, 0x16, 0x55, 0x06, 0xa5, 0x16, 0x97, 0xe6, + 0x94, 0x04, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x08, 0x49, 0x70, 0xb1, 0x17, 0x82, 0x44, 0x3d, + 0x5d, 0x24, 0x18, 0x15, 0x18, 0x35, 0x58, 0x82, 0x60, 0x5c, 0x21, 0x2b, 0x2e, 0x1e, 0x90, 0x09, + 0x41, 0xa9, 0xc5, 0x05, 0xf9, 0x79, 0xc5, 0xa9, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0xdc, 0x46, 0x62, + 0x7a, 0x70, 0x63, 0x3d, 0x42, 0x42, 0x02, 0x60, 0xb2, 0x41, 0x28, 0x6a, 0x95, 0x44, 0xb9, 0x84, + 0x51, 0xec, 0x82, 0x08, 0x1b, 0x65, 0x71, 0x09, 0xb9, 0x41, 0xfd, 0xed, 0x96, 0x5f, 0x04, 0x52, + 0x91, 0x99, 0x5a, 0x24, 0x14, 0xc2, 0xc5, 0x8d, 0xa4, 0x58, 0x48, 0x41, 0x0f, 0x39, 0x6c, 0xf4, + 0x30, 0xdd, 0x2c, 0xa5, 0x88, 0x47, 0x05, 0xd4, 0x01, 0x0c, 0x4e, 0x4e, 0x17, 0x1e, 0xca, 0x31, + 0xdc, 0x78, 0x28, 0xc7, 0xf0, 0xe1, 0xa1, 0x1c, 0x63, 0xc3, 0x23, 0x39, 0xc6, 0x15, 0x8f, 0xe4, + 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x17, 0x8f, + 0xe4, 0x18, 0x3e, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, + 0x8f, 0xe5, 0x18, 0xa2, 0x50, 0xe2, 0x25, 0x89, 0x0d, 0x1c, 0x72, 0xc6, 0x80, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xeb, 0x00, 0x1d, 0x52, 0xbe, 0x01, 0x00, 0x00, +} + +func (this *QueryResultRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*QueryResultRequest) + if !ok { + that2, ok := that.(QueryResultRequest) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.QueryID != that1.QueryID { + return false + } + if !this.HttpResponse.Equal(that1.HttpResponse) { + return false + } + return true +} +func (this *QueryResultResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*QueryResultResponse) + if !ok { + that2, ok := that.(QueryResultResponse) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + return true +} +func (this *QueryResultRequest) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&frontendv2pb.QueryResultRequest{") + s = append(s, "QueryID: "+fmt.Sprintf("%#v", this.QueryID)+",\n") + if this.HttpResponse != nil { + s = append(s, "HttpResponse: "+fmt.Sprintf("%#v", this.HttpResponse)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *QueryResultResponse) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 4) + s = append(s, "&frontendv2pb.QueryResultResponse{") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringFrontend(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// FrontendForQuerierClient is the client API for FrontendForQuerier service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type FrontendForQuerierClient interface { + QueryResult(ctx context.Context, in *QueryResultRequest, opts ...grpc.CallOption) (*QueryResultResponse, error) +} + +type frontendForQuerierClient struct { + cc *grpc.ClientConn +} + +func NewFrontendForQuerierClient(cc *grpc.ClientConn) FrontendForQuerierClient { + return &frontendForQuerierClient{cc} +} + +func (c *frontendForQuerierClient) QueryResult(ctx context.Context, in *QueryResultRequest, opts ...grpc.CallOption) (*QueryResultResponse, error) { + out := new(QueryResultResponse) + err := c.cc.Invoke(ctx, "/frontendv2pb.FrontendForQuerier/QueryResult", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// FrontendForQuerierServer is the server API for FrontendForQuerier service. +type FrontendForQuerierServer interface { + QueryResult(context.Context, *QueryResultRequest) (*QueryResultResponse, error) +} + +// UnimplementedFrontendForQuerierServer can be embedded to have forward compatible implementations. +type UnimplementedFrontendForQuerierServer struct { +} + +func (*UnimplementedFrontendForQuerierServer) QueryResult(ctx context.Context, req *QueryResultRequest) (*QueryResultResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryResult not implemented") +} + +func RegisterFrontendForQuerierServer(s *grpc.Server, srv FrontendForQuerierServer) { + s.RegisterService(&_FrontendForQuerier_serviceDesc, srv) +} + +func _FrontendForQuerier_QueryResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueryResultRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(FrontendForQuerierServer).QueryResult(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/frontendv2pb.FrontendForQuerier/QueryResult", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(FrontendForQuerierServer).QueryResult(ctx, req.(*QueryResultRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _FrontendForQuerier_serviceDesc = grpc.ServiceDesc{ + ServiceName: "frontendv2pb.FrontendForQuerier", + HandlerType: (*FrontendForQuerierServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "QueryResult", + Handler: _FrontendForQuerier_QueryResult_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "frontend.proto", +} + +func (m *QueryResultRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResultRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueryResultRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.HttpResponse != nil { + { + size, err := m.HttpResponse.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintFrontend(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.QueryID != 0 { + i = encodeVarintFrontend(dAtA, i, uint64(m.QueryID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *QueryResultResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResultResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueryResultResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + return len(dAtA) - i, nil +} + +func encodeVarintFrontend(dAtA []byte, offset int, v uint64) int { + offset -= sovFrontend(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *QueryResultRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.QueryID != 0 { + n += 1 + sovFrontend(uint64(m.QueryID)) + } + if m.HttpResponse != nil { + l = m.HttpResponse.Size() + n += 1 + l + sovFrontend(uint64(l)) + } + return n +} + +func (m *QueryResultResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + return n +} + +func sovFrontend(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozFrontend(x uint64) (n int) { + return sovFrontend(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *QueryResultRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&QueryResultRequest{`, + `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, + `HttpResponse:` + strings.Replace(fmt.Sprintf("%v", this.HttpResponse), "HTTPResponse", "httpgrpc.HTTPResponse", 1) + `,`, + `}`, + }, "") + return s +} +func (this *QueryResultResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&QueryResultResponse{`, + `}`, + }, "") + return s +} +func valueToStringFrontend(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *QueryResultRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFrontend + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResultRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResultRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) + } + m.QueryID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFrontend + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.QueryID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HttpResponse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFrontend + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthFrontend + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFrontend + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.HttpResponse == nil { + m.HttpResponse = &httpgrpc.HTTPResponse{} + } + if err := m.HttpResponse.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipFrontend(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthFrontend + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFrontend + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResultResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFrontend + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResultResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResultResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipFrontend(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthFrontend + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFrontend + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipFrontend(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowFrontend + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowFrontend + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowFrontend + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthFrontend + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthFrontend + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowFrontend + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipFrontend(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthFrontend + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthFrontend = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowFrontend = fmt.Errorf("proto: integer overflow") +) diff --git a/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.proto b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.proto new file mode 100644 index 000000000000..367f352afd37 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb/frontend.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package frontendv2pb; + +option go_package = "frontendv2pb"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/weaveworks/common/httpgrpc/httpgrpc.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +// Frontend interface exposed to Queriers. Used by queriers to report back the result of the query. +service FrontendForQuerier { + rpc QueryResult (QueryResultRequest) returns (QueryResultResponse) { }; +} + +message QueryResultRequest { + uint64 queryID = 1; + httpgrpc.HTTPResponse httpResponse = 2; + + // There is no userID field here, because Querier puts userID into the context when + // calling QueryResult, and that is where Frontend expects to find it. +} + +message QueryResultResponse { } diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go index bcc94a7c60bc..b71ec16a19cb 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go @@ -447,7 +447,9 @@ func (m *QueryStreamResponse) GetTimeseries() []TimeSeries { } type LabelValuesRequest struct { - LabelName string `protobuf:"bytes,1,opt,name=label_name,json=labelName,proto3" json:"label_name,omitempty"` + LabelName string `protobuf:"bytes,1,opt,name=label_name,json=labelName,proto3" json:"label_name,omitempty"` + StartTimestampMs int64 `protobuf:"varint,2,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` + EndTimestampMs int64 `protobuf:"varint,3,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` } func (m *LabelValuesRequest) Reset() { *m = LabelValuesRequest{} } @@ -489,6 +491,20 @@ func (m *LabelValuesRequest) GetLabelName() string { return "" } +func (m *LabelValuesRequest) GetStartTimestampMs() int64 { + if m != nil { + return m.StartTimestampMs + } + return 0 +} + +func (m *LabelValuesRequest) GetEndTimestampMs() int64 { + if m != nil { + return m.EndTimestampMs + } + return 0 +} + type LabelValuesResponse struct { LabelValues []string `protobuf:"bytes,1,rep,name=label_values,json=labelValues,proto3" json:"label_values,omitempty"` } @@ -533,6 +549,8 @@ func (m *LabelValuesResponse) GetLabelValues() []string { } type LabelNamesRequest struct { + StartTimestampMs int64 `protobuf:"varint,1,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` + EndTimestampMs int64 `protobuf:"varint,2,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` } func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } @@ -567,6 +585,20 @@ func (m *LabelNamesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_LabelNamesRequest proto.InternalMessageInfo +func (m *LabelNamesRequest) GetStartTimestampMs() int64 { + if m != nil { + return m.StartTimestampMs + } + return 0 +} + +func (m *LabelNamesRequest) GetEndTimestampMs() int64 { + if m != nil { + return m.EndTimestampMs + } + return 0 +} + type LabelNamesResponse struct { LabelNames []string `protobuf:"bytes,1,rep,name=label_names,json=labelNames,proto3" json:"label_names,omitempty"` } @@ -1606,98 +1638,99 @@ func init() { func init() { proto.RegisterFile("cortex.proto", fileDescriptor_893a47d0a749d749) } var fileDescriptor_893a47d0a749d749 = []byte{ - // 1456 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0xbb, 0x6f, 0x1b, 0x47, - 0x13, 0xbf, 0x15, 0x1f, 0x12, 0x87, 0x14, 0x75, 0x5a, 0xc9, 0x36, 0x4d, 0xe3, 0x3b, 0xda, 0x0b, - 0xd8, 0x9f, 0xf0, 0x7d, 0xb1, 0xec, 0xc8, 0x70, 0xa2, 0x22, 0x81, 0x41, 0xd9, 0x94, 0xcc, 0x44, - 0xa4, 0xe4, 0x25, 0x15, 0x27, 0x01, 0x02, 0xe2, 0x44, 0xae, 0xa4, 0x43, 0xee, 0x8e, 0xf4, 0x3d, - 0x82, 0xa8, 0x08, 0x10, 0x20, 0x65, 0x8a, 0xb8, 0xcc, 0x9f, 0x90, 0x3a, 0x4d, 0xfa, 0x54, 0x2e, - 0x5d, 0x1a, 0x29, 0x8c, 0x58, 0x6e, 0xd2, 0xc5, 0xc8, 0x5f, 0x10, 0xec, 0xe3, 0x8e, 0x77, 0x34, - 0x95, 0x38, 0x0f, 0x77, 0xb7, 0x33, 0xbf, 0xf9, 0xed, 0xec, 0xec, 0xcc, 0xec, 0x1c, 0x94, 0xfa, - 0x43, 0x2f, 0x60, 0x9f, 0xaf, 0x8e, 0xbc, 0x61, 0x30, 0xc4, 0x79, 0xb9, 0xaa, 0x5e, 0x3d, 0xb4, - 0x82, 0xa3, 0x70, 0x7f, 0xb5, 0x3f, 0x74, 0xae, 0x1d, 0x0e, 0x0f, 0x87, 0xd7, 0x84, 0x7a, 0x3f, - 0x3c, 0x10, 0x2b, 0xb1, 0x10, 0x5f, 0xd2, 0x8c, 0xfc, 0x86, 0xa0, 0x74, 0xdf, 0xb3, 0x02, 0x46, - 0xd9, 0x83, 0x90, 0xf9, 0x01, 0x6e, 0x03, 0x04, 0x96, 0xc3, 0x7c, 0xe6, 0x59, 0xcc, 0xaf, 0xa0, - 0x8b, 0x99, 0x95, 0xe2, 0x1a, 0x5e, 0x55, 0x5b, 0x75, 0x2d, 0x87, 0x75, 0x84, 0x66, 0xa3, 0xfa, - 0xe8, 0x69, 0x4d, 0xfb, 0xe9, 0x69, 0x0d, 0xef, 0x7a, 0xcc, 0xb4, 0xed, 0x61, 0xbf, 0x1b, 0x5b, - 0xd1, 0x04, 0x03, 0x7e, 0x1b, 0xf2, 0x9d, 0x61, 0xe8, 0xf5, 0x59, 0x65, 0xe6, 0x22, 0x5a, 0x29, - 0xaf, 0xd5, 0x22, 0xae, 0xe4, 0xae, 0xab, 0x12, 0xd2, 0x70, 0x43, 0x87, 0x2a, 0x38, 0x5e, 0x87, - 0x39, 0x87, 0x05, 0xe6, 0xc0, 0x0c, 0xcc, 0x4a, 0x46, 0xb8, 0x71, 0x36, 0x32, 0x6d, 0xb1, 0xc0, - 0xb3, 0xfa, 0x2d, 0xa5, 0xdd, 0xc8, 0x3e, 0x7a, 0x5a, 0x43, 0x34, 0x46, 0x93, 0x1a, 0xc0, 0x98, - 0x0f, 0xcf, 0x42, 0xa6, 0xbe, 0xdb, 0xd4, 0x35, 0x3c, 0x07, 0x59, 0xba, 0xb7, 0xdd, 0xd0, 0x11, - 0x59, 0x80, 0x79, 0xb5, 0xbb, 0x3f, 0x1a, 0xba, 0x3e, 0x23, 0xef, 0x42, 0x91, 0x32, 0x73, 0x10, - 0xc5, 0x60, 0x15, 0x66, 0x1f, 0x84, 0xc9, 0x00, 0x2c, 0x47, 0x3b, 0xdf, 0x0b, 0x99, 0x77, 0xac, - 0x60, 0x34, 0x02, 0x91, 0x5b, 0x50, 0x92, 0xe6, 0x92, 0x0e, 0x5f, 0x83, 0x59, 0x8f, 0xf9, 0xa1, - 0x1d, 0x44, 0xf6, 0x67, 0x26, 0xec, 0x25, 0x8e, 0x46, 0x28, 0xf2, 0x2d, 0x82, 0x52, 0x92, 0x1a, - 0xbf, 0x01, 0xd8, 0x0f, 0x4c, 0x2f, 0xe8, 0x89, 0x48, 0x06, 0xa6, 0x33, 0xea, 0x39, 0x9c, 0x0c, - 0xad, 0x64, 0xa8, 0x2e, 0x34, 0xdd, 0x48, 0xd1, 0xf2, 0xf1, 0x0a, 0xe8, 0xcc, 0x1d, 0xa4, 0xb1, - 0x33, 0x02, 0x5b, 0x66, 0xee, 0x20, 0x89, 0xbc, 0x0e, 0x73, 0x8e, 0x19, 0xf4, 0x8f, 0x98, 0xe7, - 0xab, 0xa0, 0xc6, 0x47, 0xdb, 0x36, 0xf7, 0x99, 0xdd, 0x92, 0x4a, 0x1a, 0xa3, 0x48, 0x13, 0xe6, - 0x53, 0x4e, 0xe3, 0xf5, 0x57, 0x4c, 0x10, 0x7e, 0x2b, 0x5a, 0x32, 0x15, 0xc8, 0x43, 0x04, 0x4b, - 0x82, 0xab, 0x13, 0x78, 0xcc, 0x74, 0x62, 0xc6, 0x5b, 0x50, 0xec, 0x1f, 0x85, 0xee, 0xa7, 0x29, - 0xca, 0x73, 0x2f, 0x53, 0xde, 0xe6, 0x20, 0xc5, 0x9b, 0xb4, 0x98, 0x70, 0x69, 0xe6, 0x2f, 0xb8, - 0x74, 0x03, 0xb0, 0x38, 0xf7, 0x07, 0xa6, 0x1d, 0x32, 0x3f, 0x8a, 0xfe, 0x7f, 0x00, 0x6c, 0x2e, - 0xed, 0xb9, 0xa6, 0xc3, 0x44, 0xd4, 0x0b, 0xb4, 0x20, 0x24, 0x6d, 0xd3, 0x61, 0x64, 0x1d, 0x96, - 0x52, 0x46, 0xea, 0x18, 0x97, 0xa0, 0x24, 0xad, 0x3e, 0x13, 0x72, 0x71, 0x8e, 0x02, 0x2d, 0xda, - 0x63, 0x28, 0x59, 0x82, 0xc5, 0xed, 0x88, 0x26, 0xda, 0x8d, 0xdc, 0x54, 0x3e, 0x28, 0xa1, 0x62, - 0xab, 0x41, 0x71, 0xec, 0x43, 0x44, 0x06, 0xb1, 0x13, 0x3e, 0xc1, 0xa0, 0xef, 0xf9, 0xcc, 0xeb, - 0x04, 0x66, 0x10, 0x53, 0xfd, 0x80, 0x60, 0x31, 0x21, 0x54, 0x54, 0x97, 0xa1, 0x6c, 0xb9, 0x87, - 0xcc, 0x0f, 0xac, 0xa1, 0xdb, 0xf3, 0xcc, 0x40, 0x1e, 0x09, 0xd1, 0xf9, 0x58, 0x4a, 0xcd, 0x80, - 0xf1, 0x53, 0xbb, 0xa1, 0xd3, 0x8b, 0xa3, 0x88, 0x56, 0xb2, 0xb4, 0xe0, 0x86, 0x8e, 0x0c, 0x1e, - 0x4f, 0x49, 0x73, 0x64, 0xf5, 0x26, 0x98, 0x32, 0x82, 0x49, 0x37, 0x47, 0x56, 0x33, 0x45, 0xb6, - 0x0a, 0x4b, 0x5e, 0x68, 0xb3, 0x49, 0x78, 0x56, 0xc0, 0x17, 0xb9, 0x2a, 0x85, 0x27, 0x9f, 0xc0, - 0x12, 0x77, 0xbc, 0x79, 0x27, 0xed, 0xfa, 0x39, 0x98, 0x0d, 0x7d, 0xe6, 0xf5, 0xac, 0x81, 0xba, - 0x86, 0x3c, 0x5f, 0x36, 0x07, 0xf8, 0x2a, 0x64, 0x45, 0x67, 0xe0, 0x6e, 0x16, 0xd7, 0xce, 0x47, - 0x97, 0xfd, 0xd2, 0xe1, 0xa9, 0x80, 0x91, 0x2d, 0xc0, 0x5c, 0xe5, 0xa7, 0xd9, 0xdf, 0x84, 0x9c, - 0xcf, 0x05, 0x2a, 0xe5, 0x2e, 0x24, 0x59, 0x26, 0x3c, 0xa1, 0x12, 0x49, 0xbe, 0x47, 0x60, 0xc8, - 0xf6, 0xe3, 0x6f, 0x0e, 0xbd, 0x64, 0xcd, 0xf8, 0xaf, 0xbb, 0x76, 0xd7, 0xa1, 0x14, 0x55, 0x65, - 0xcf, 0x67, 0x81, 0xaa, 0xdf, 0x33, 0xd3, 0xea, 0xd7, 0xa7, 0xc5, 0x08, 0xda, 0x61, 0x01, 0x69, - 0x42, 0xed, 0x54, 0x9f, 0x55, 0x28, 0xae, 0x40, 0xde, 0x11, 0x10, 0x15, 0x8b, 0x72, 0xba, 0xd7, - 0x52, 0xa5, 0x25, 0x15, 0x38, 0xab, 0xa8, 0xa2, 0xf6, 0x1b, 0xe5, 0x5e, 0x0b, 0xce, 0xbd, 0xa4, - 0x51, 0xe4, 0x6b, 0x89, 0x56, 0x8e, 0xfe, 0xa8, 0x95, 0x27, 0x9a, 0xf8, 0x8f, 0x08, 0x16, 0x26, - 0x4a, 0x9f, 0xc7, 0xea, 0xc0, 0x1b, 0x3a, 0x2a, 0xa9, 0x92, 0x69, 0x51, 0xe6, 0xf2, 0xa6, 0x12, - 0x37, 0x07, 0xc9, 0xbc, 0x99, 0x49, 0xe5, 0xcd, 0x2d, 0xc8, 0x8b, 0x1a, 0x8a, 0xda, 0xdf, 0x62, - 0x2a, 0x7c, 0xbb, 0xa6, 0xe5, 0x6d, 0x2c, 0xab, 0x97, 0xad, 0x24, 0x44, 0xf5, 0x81, 0x39, 0x0a, - 0x98, 0x47, 0x95, 0x19, 0xfe, 0x3f, 0xe4, 0x65, 0xeb, 0xa9, 0x64, 0x05, 0xc1, 0x7c, 0x44, 0x90, - 0xec, 0x4e, 0x0a, 0x42, 0xbe, 0x41, 0x90, 0x93, 0xae, 0xbf, 0xae, 0xa4, 0xa8, 0xc2, 0x1c, 0x73, - 0xfb, 0xc3, 0x81, 0xe5, 0x1e, 0x8a, 0x5a, 0xcc, 0xd1, 0x78, 0x8d, 0xb1, 0xaa, 0x11, 0x5e, 0x74, - 0x25, 0x55, 0x08, 0x15, 0x38, 0xdb, 0xf5, 0x4c, 0xd7, 0x3f, 0x60, 0x9e, 0x70, 0x2c, 0xce, 0x00, - 0xf2, 0x05, 0xc0, 0x38, 0xde, 0x89, 0x38, 0xa1, 0xbf, 0x17, 0xa7, 0x55, 0x98, 0xf5, 0x4d, 0x67, - 0x64, 0xc7, 0x0d, 0x39, 0xce, 0xa8, 0x8e, 0x10, 0xab, 0x48, 0x45, 0x20, 0x72, 0x13, 0x0a, 0x31, - 0x35, 0xf7, 0x3c, 0x6e, 0xbd, 0x25, 0x2a, 0xbe, 0xf1, 0x32, 0xe4, 0x44, 0x63, 0x15, 0x81, 0x28, - 0x51, 0xb9, 0x20, 0x75, 0xc8, 0x4b, 0xbe, 0xb1, 0x5e, 0x36, 0x37, 0xb9, 0xe0, 0x4d, 0x79, 0x4a, - 0x14, 0x8b, 0xc1, 0x38, 0x84, 0xa4, 0x0e, 0xf3, 0xa9, 0x9a, 0x48, 0x3d, 0x92, 0xe8, 0x95, 0x1e, - 0xc9, 0xaf, 0x67, 0xa0, 0x9c, 0xce, 0x64, 0x7c, 0x13, 0xb2, 0xc1, 0xf1, 0x48, 0x7a, 0x53, 0x5e, - 0xbb, 0x34, 0x3d, 0xdf, 0xd5, 0xb2, 0x7b, 0x3c, 0x62, 0x54, 0xc0, 0x79, 0xdb, 0x97, 0x95, 0x26, - 0xdf, 0x1e, 0x99, 0xbc, 0x20, 0x45, 0xbc, 0xef, 0xf3, 0xd0, 0x1c, 0x31, 0x7b, 0x24, 0x2e, 0xb5, - 0x40, 0xc5, 0x37, 0x97, 0x85, 0xae, 0x15, 0x54, 0x72, 0x52, 0xc6, 0xbf, 0xc9, 0x31, 0xc0, 0x98, - 0x1c, 0x17, 0x61, 0x76, 0xaf, 0xfd, 0x7e, 0x7b, 0xe7, 0x7e, 0x5b, 0xd7, 0xf8, 0xe2, 0xf6, 0xce, - 0x5e, 0xbb, 0xdb, 0xa0, 0x3a, 0xc2, 0x05, 0xc8, 0x6d, 0xd5, 0xf7, 0xb6, 0x1a, 0xfa, 0x0c, 0x9e, - 0x87, 0xc2, 0xdd, 0x66, 0xa7, 0xbb, 0xb3, 0x45, 0xeb, 0x2d, 0x3d, 0x83, 0x31, 0x94, 0x85, 0x66, - 0x2c, 0xcb, 0x72, 0xd3, 0xce, 0x5e, 0xab, 0x55, 0xa7, 0x1f, 0xe9, 0x39, 0x3e, 0x50, 0x35, 0xdb, - 0x9b, 0x3b, 0x7a, 0x1e, 0x97, 0x60, 0xae, 0xd3, 0xad, 0x77, 0x1b, 0x9d, 0x46, 0x57, 0x9f, 0x25, - 0x4d, 0xc8, 0xcb, 0xad, 0xff, 0x71, 0x16, 0x91, 0x1e, 0x94, 0x92, 0x21, 0xc7, 0x97, 0x53, 0x51, - 0x8d, 0xe9, 0x84, 0x3a, 0x11, 0xc5, 0x28, 0x7f, 0x64, 0xf8, 0x26, 0xf2, 0x27, 0x23, 0x84, 0x2a, - 0x7f, 0xbe, 0x42, 0x50, 0x1e, 0xa7, 0xfd, 0xa6, 0x65, 0xb3, 0x7f, 0xa3, 0xcb, 0x54, 0x61, 0xee, - 0xc0, 0xb2, 0x99, 0xf0, 0x41, 0x6e, 0x17, 0xaf, 0xa7, 0x55, 0xe5, 0xff, 0xde, 0x83, 0x42, 0x7c, - 0x04, 0x7e, 0x23, 0x8d, 0x7b, 0x7b, 0xf5, 0x6d, 0x5d, 0xe3, 0x37, 0xd2, 0xde, 0xe9, 0xf6, 0xe4, - 0x12, 0xe1, 0x05, 0x28, 0xd2, 0xc6, 0x56, 0xe3, 0xc3, 0x5e, 0xab, 0xde, 0xbd, 0x7d, 0x57, 0x9f, - 0xe1, 0x57, 0x24, 0x05, 0xed, 0x1d, 0x25, 0xcb, 0xac, 0xfd, 0x9a, 0x83, 0xb9, 0xc8, 0x47, 0x9e, - 0x85, 0xbb, 0xa1, 0x7f, 0x84, 0x97, 0xa7, 0x4d, 0xdd, 0xd5, 0x33, 0x13, 0x52, 0xd5, 0x09, 0x34, - 0xfc, 0x16, 0xe4, 0xc4, 0xa0, 0x86, 0xa7, 0x0e, 0xbe, 0xd5, 0xe9, 0xe3, 0x2c, 0xd1, 0xf0, 0x1d, - 0x28, 0x26, 0x06, 0xbc, 0x53, 0xac, 0x2f, 0xa4, 0xa4, 0xe9, 0x59, 0x90, 0x68, 0xd7, 0x11, 0xbe, - 0x0b, 0xc5, 0xc4, 0x7c, 0x85, 0xab, 0xa9, 0xa4, 0x49, 0x4d, 0x6a, 0x63, 0xae, 0x29, 0x03, 0x19, - 0xd1, 0x70, 0x03, 0x60, 0x3c, 0x5a, 0xe1, 0xf3, 0x29, 0x70, 0x72, 0x06, 0xab, 0x56, 0xa7, 0xa9, - 0x62, 0x9a, 0x0d, 0x28, 0xc4, 0x83, 0x05, 0xae, 0x4c, 0x99, 0x35, 0x24, 0xc9, 0xe9, 0x53, 0x08, - 0xd1, 0xf0, 0x26, 0x94, 0xea, 0xb6, 0xfd, 0x2a, 0x34, 0xd5, 0xa4, 0xc6, 0x9f, 0xe4, 0xb1, 0xe3, - 0x67, 0x76, 0xf2, 0x2d, 0xc7, 0x57, 0xd2, 0x4d, 0xe6, 0xb4, 0x01, 0xa5, 0xfa, 0xdf, 0x3f, 0xc5, - 0xc5, 0xbb, 0x75, 0x61, 0x61, 0xe2, 0x51, 0xc7, 0xc6, 0x84, 0xf5, 0xc4, 0x1c, 0x50, 0xad, 0x9d, - 0xaa, 0x8f, 0x59, 0x5b, 0x50, 0x4e, 0x3f, 0x42, 0xf8, 0xb4, 0x69, 0xbf, 0x1a, 0xef, 0x76, 0xca, - 0xab, 0xa5, 0xad, 0xa0, 0x8d, 0x77, 0x1e, 0x3f, 0x33, 0xb4, 0x27, 0xcf, 0x0c, 0xed, 0xc5, 0x33, - 0x03, 0x7d, 0x79, 0x62, 0xa0, 0xef, 0x4e, 0x0c, 0xf4, 0xe8, 0xc4, 0x40, 0x8f, 0x4f, 0x0c, 0xf4, - 0xf3, 0x89, 0x81, 0x7e, 0x39, 0x31, 0xb4, 0x17, 0x27, 0x06, 0x7a, 0xf8, 0xdc, 0xd0, 0x1e, 0x3f, - 0x37, 0xb4, 0x27, 0xcf, 0x0d, 0xed, 0xe3, 0x7c, 0xdf, 0xb6, 0x98, 0x1b, 0xec, 0xe7, 0xc5, 0x8f, - 0xf0, 0x8d, 0xdf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x3b, 0x08, 0x1c, 0x63, 0x4f, 0x0f, 0x00, 0x00, + // 1469 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0xcf, 0x6f, 0x1b, 0x45, + 0x14, 0xde, 0xf1, 0xaf, 0xc4, 0xcf, 0x8e, 0xb3, 0x99, 0xa4, 0xad, 0xeb, 0x8a, 0x75, 0x3b, 0x52, + 0x4b, 0x04, 0x34, 0x2d, 0x41, 0x85, 0x1c, 0x40, 0x95, 0xd3, 0x3a, 0xa9, 0x21, 0x76, 0xd2, 0xb1, + 0x43, 0x01, 0x09, 0x59, 0x1b, 0x7b, 0x92, 0xac, 0xba, 0xbb, 0x76, 0xf7, 0x07, 0x22, 0x07, 0x24, + 0x24, 0x8e, 0x3d, 0xd0, 0x23, 0x7f, 0x02, 0x67, 0x2e, 0xdc, 0x39, 0xf5, 0xd8, 0x63, 0xc5, 0xa1, + 0xa2, 0xee, 0x85, 0x1b, 0x15, 0x7f, 0x01, 0xda, 0x99, 0xd9, 0xf5, 0xae, 0x6b, 0x43, 0x0b, 0xf4, + 0xe6, 0x79, 0xef, 0x9b, 0x6f, 0xde, 0x7e, 0xf3, 0xe6, 0xbd, 0x67, 0x28, 0xf6, 0x06, 0x8e, 0xc7, + 0xbe, 0x5e, 0x1b, 0x3a, 0x03, 0x6f, 0x80, 0x73, 0x62, 0x55, 0xb9, 0x7c, 0x64, 0x78, 0xc7, 0xfe, + 0xc1, 0x5a, 0x6f, 0x60, 0x5d, 0x39, 0x1a, 0x1c, 0x0d, 0xae, 0x70, 0xf7, 0x81, 0x7f, 0xc8, 0x57, + 0x7c, 0xc1, 0x7f, 0x89, 0x6d, 0xe4, 0x4f, 0x04, 0xc5, 0x3b, 0x8e, 0xe1, 0x31, 0xca, 0xee, 0xf9, + 0xcc, 0xf5, 0x70, 0x0b, 0xc0, 0x33, 0x2c, 0xe6, 0x32, 0xc7, 0x60, 0x6e, 0x19, 0x9d, 0x4f, 0xaf, + 0x16, 0xd6, 0xf1, 0x9a, 0x3c, 0xaa, 0x63, 0x58, 0xac, 0xcd, 0x3d, 0x9b, 0x95, 0x87, 0x4f, 0xaa, + 0xca, 0xaf, 0x4f, 0xaa, 0x78, 0xcf, 0x61, 0xba, 0x69, 0x0e, 0x7a, 0x9d, 0x68, 0x17, 0x8d, 0x31, + 0xe0, 0x0f, 0x20, 0xd7, 0x1e, 0xf8, 0x4e, 0x8f, 0x95, 0x53, 0xe7, 0xd1, 0x6a, 0x69, 0xbd, 0x1a, + 0x72, 0xc5, 0x4f, 0x5d, 0x13, 0x90, 0xba, 0xed, 0x5b, 0x54, 0xc2, 0xf1, 0x06, 0xcc, 0x5b, 0xcc, + 0xd3, 0xfb, 0xba, 0xa7, 0x97, 0xd3, 0x3c, 0x8c, 0xd3, 0xe1, 0xd6, 0x26, 0xf3, 0x1c, 0xa3, 0xd7, + 0x94, 0xde, 0xcd, 0xcc, 0xc3, 0x27, 0x55, 0x44, 0x23, 0x34, 0xa9, 0x02, 0x8c, 0xf9, 0xf0, 0x1c, + 0xa4, 0x6b, 0x7b, 0x0d, 0x55, 0xc1, 0xf3, 0x90, 0xa1, 0xfb, 0x3b, 0x75, 0x15, 0x91, 0x45, 0x58, + 0x90, 0xa7, 0xbb, 0xc3, 0x81, 0xed, 0x32, 0xf2, 0x11, 0x14, 0x28, 0xd3, 0xfb, 0xa1, 0x06, 0x6b, + 0x30, 0x77, 0xcf, 0x8f, 0x0b, 0xb0, 0x12, 0x9e, 0x7c, 0xdb, 0x67, 0xce, 0x89, 0x84, 0xd1, 0x10, + 0x44, 0xae, 0x43, 0x51, 0x6c, 0x17, 0x74, 0xf8, 0x0a, 0xcc, 0x39, 0xcc, 0xf5, 0x4d, 0x2f, 0xdc, + 0x7f, 0x6a, 0x62, 0xbf, 0xc0, 0xd1, 0x10, 0x45, 0x7e, 0x40, 0x50, 0x8c, 0x53, 0xe3, 0x77, 0x00, + 0xbb, 0x9e, 0xee, 0x78, 0x5d, 0xae, 0xa4, 0xa7, 0x5b, 0xc3, 0xae, 0x15, 0x90, 0xa1, 0xd5, 0x34, + 0x55, 0xb9, 0xa7, 0x13, 0x3a, 0x9a, 0x2e, 0x5e, 0x05, 0x95, 0xd9, 0xfd, 0x24, 0x36, 0xc5, 0xb1, + 0x25, 0x66, 0xf7, 0xe3, 0xc8, 0xab, 0x30, 0x6f, 0xe9, 0x5e, 0xef, 0x98, 0x39, 0xae, 0x14, 0x35, + 0xfa, 0xb4, 0x1d, 0xfd, 0x80, 0x99, 0x4d, 0xe1, 0xa4, 0x11, 0x8a, 0x34, 0x60, 0x21, 0x11, 0x34, + 0xde, 0x78, 0xc9, 0x04, 0x09, 0x6e, 0x45, 0x89, 0xa7, 0x02, 0x79, 0x80, 0x60, 0x99, 0x73, 0xb5, + 0x3d, 0x87, 0xe9, 0x56, 0xc4, 0x78, 0x1d, 0x0a, 0xbd, 0x63, 0xdf, 0xbe, 0x9b, 0xa0, 0x3c, 0xf3, + 0x22, 0xe5, 0x8d, 0x00, 0x24, 0x79, 0xe3, 0x3b, 0x26, 0x42, 0x4a, 0xbd, 0x42, 0x48, 0xf7, 0x11, + 0x60, 0xfe, 0xe1, 0x9f, 0xea, 0xa6, 0xcf, 0xdc, 0x50, 0xfe, 0x37, 0x00, 0xcc, 0xc0, 0xda, 0xb5, + 0x75, 0x8b, 0x71, 0xd9, 0xf3, 0x34, 0xcf, 0x2d, 0x2d, 0xdd, 0x62, 0x33, 0x6e, 0x27, 0xf5, 0x0a, + 0xb7, 0x93, 0x9e, 0x76, 0x3b, 0x64, 0x03, 0x96, 0x13, 0xc1, 0x48, 0x7d, 0x2e, 0x40, 0x51, 0x44, + 0xf3, 0x15, 0xb7, 0x73, 0x81, 0xf2, 0xb4, 0x60, 0x8e, 0xa1, 0xe4, 0x2e, 0x2c, 0xed, 0x84, 0xe1, + 0xb9, 0xaf, 0x39, 0x89, 0xc8, 0x35, 0xa9, 0x99, 0x3c, 0x4c, 0x46, 0x59, 0x85, 0xc2, 0x58, 0xb3, + 0x30, 0x48, 0x88, 0x44, 0x73, 0x09, 0x06, 0x75, 0xdf, 0x65, 0x4e, 0xdb, 0xd3, 0xbd, 0x30, 0x44, + 0xf2, 0x33, 0x82, 0xa5, 0x98, 0x51, 0x52, 0x5d, 0x84, 0x92, 0x61, 0x1f, 0x31, 0xd7, 0x33, 0x06, + 0x76, 0xd7, 0xd1, 0x3d, 0x71, 0x05, 0x88, 0x2e, 0x44, 0x56, 0xaa, 0x7b, 0x2c, 0xb8, 0x25, 0xdb, + 0xb7, 0xba, 0xd1, 0xb5, 0xa3, 0xd5, 0x0c, 0xcd, 0xdb, 0xbe, 0x25, 0x6e, 0x3b, 0xf8, 0x7c, 0x7d, + 0x68, 0x74, 0x27, 0x98, 0xd2, 0x9c, 0x49, 0xd5, 0x87, 0x46, 0x23, 0x41, 0xb6, 0x06, 0xcb, 0x8e, + 0x6f, 0xb2, 0x49, 0x78, 0x86, 0xc3, 0x97, 0x02, 0x57, 0x02, 0x4f, 0xbe, 0x84, 0xe5, 0x20, 0xf0, + 0xc6, 0xcd, 0x64, 0xe8, 0x67, 0x60, 0xce, 0x77, 0x99, 0xd3, 0x35, 0xfa, 0x32, 0x6d, 0x72, 0xc1, + 0xb2, 0xd1, 0xc7, 0x97, 0x21, 0xc3, 0x4b, 0x59, 0x10, 0x66, 0x61, 0xfd, 0x6c, 0x98, 0x9d, 0x2f, + 0x7c, 0x3c, 0xe5, 0x30, 0xb2, 0x0d, 0x38, 0x70, 0xb9, 0x49, 0xf6, 0x77, 0x21, 0xeb, 0x06, 0x06, + 0xf9, 0x46, 0xce, 0xc5, 0x59, 0x26, 0x22, 0xa1, 0x02, 0x49, 0x7e, 0x42, 0xa0, 0x89, 0x7a, 0xe9, + 0x6e, 0x0d, 0x9c, 0xf8, 0x23, 0x7f, 0xdd, 0x79, 0x82, 0x37, 0xa0, 0x18, 0x96, 0x91, 0xae, 0xcb, + 0x3c, 0x59, 0x70, 0x4e, 0x4d, 0x2b, 0x38, 0x2e, 0x2d, 0x84, 0xd0, 0x36, 0xf3, 0x48, 0x03, 0xaa, + 0x33, 0x63, 0x96, 0x52, 0x5c, 0x82, 0x9c, 0xc5, 0x21, 0x52, 0x8b, 0x52, 0xb2, 0x39, 0x50, 0xe9, + 0x25, 0x65, 0x38, 0x2d, 0xa9, 0xc2, 0x7e, 0x11, 0xe6, 0x5e, 0x13, 0xce, 0xbc, 0xe0, 0x91, 0xe4, + 0xeb, 0xb1, 0xde, 0x83, 0xfe, 0xae, 0xf7, 0xc4, 0xba, 0xce, 0x2f, 0x08, 0x16, 0x27, 0x6a, 0x55, + 0xa0, 0xd5, 0xa1, 0x33, 0xb0, 0x64, 0x52, 0xc5, 0xd3, 0xa2, 0x14, 0xd8, 0x1b, 0xd2, 0xdc, 0xe8, + 0xc7, 0xf3, 0x26, 0x95, 0xc8, 0x9b, 0xeb, 0x90, 0xe3, 0x6f, 0x28, 0xac, 0xd7, 0x4b, 0x09, 0xf9, + 0xf6, 0x74, 0xc3, 0xd9, 0x5c, 0x91, 0xad, 0xb8, 0xc8, 0x4d, 0xb5, 0xbe, 0x3e, 0xf4, 0x98, 0x43, + 0xe5, 0x36, 0xfc, 0x36, 0xe4, 0x44, 0xad, 0x2c, 0x67, 0x38, 0xc1, 0x42, 0x48, 0x10, 0x2f, 0xa7, + 0x12, 0x42, 0xbe, 0x47, 0x90, 0x15, 0xa1, 0xbf, 0xae, 0xa4, 0xa8, 0xc0, 0x3c, 0xb3, 0x7b, 0x83, + 0xbe, 0x61, 0x1f, 0xf1, 0xb7, 0x98, 0xa5, 0xd1, 0x1a, 0x63, 0xf9, 0x46, 0x82, 0x47, 0x57, 0x94, + 0x0f, 0xa1, 0x0c, 0xa7, 0x3b, 0x8e, 0x6e, 0xbb, 0x87, 0xcc, 0xe1, 0x81, 0x45, 0x19, 0x40, 0xbe, + 0x01, 0x18, 0xeb, 0x1d, 0xd3, 0x09, 0xfd, 0x3b, 0x9d, 0xd6, 0x60, 0xce, 0xd5, 0xad, 0xa1, 0x19, + 0x75, 0x90, 0x28, 0xa3, 0xda, 0xdc, 0x2c, 0x95, 0x0a, 0x41, 0xe4, 0x1a, 0xe4, 0x23, 0xea, 0x20, + 0xf2, 0xa8, 0x55, 0x14, 0x29, 0xff, 0x8d, 0x57, 0x20, 0xcb, 0x0b, 0x36, 0x17, 0xa2, 0x48, 0xc5, + 0x82, 0xd4, 0x20, 0x27, 0xf8, 0xc6, 0x7e, 0x51, 0xdc, 0xc4, 0x22, 0x28, 0xf6, 0x53, 0x54, 0x2c, + 0x78, 0xb1, 0xfa, 0x5b, 0x83, 0x85, 0xc4, 0x9b, 0x48, 0x74, 0x75, 0xf4, 0x52, 0x5d, 0xfd, 0x7e, + 0x0a, 0x4a, 0xc9, 0x4c, 0xc6, 0xd7, 0x20, 0xe3, 0x9d, 0x0c, 0x45, 0x34, 0xa5, 0xf5, 0x0b, 0xd3, + 0xf3, 0x5d, 0x2e, 0x3b, 0x27, 0x43, 0x46, 0x39, 0x3c, 0x28, 0xfb, 0xe2, 0xa5, 0x89, 0x5e, 0x29, + 0x92, 0x17, 0x84, 0x89, 0x37, 0x4b, 0x0c, 0x99, 0x63, 0x66, 0x0e, 0xf9, 0xa5, 0xe6, 0x29, 0xff, + 0x1d, 0xd8, 0x7c, 0xdb, 0xf0, 0xca, 0x59, 0x61, 0x0b, 0x7e, 0x93, 0x13, 0x80, 0x31, 0x39, 0x2e, + 0xc0, 0xdc, 0x7e, 0xeb, 0x93, 0xd6, 0xee, 0x9d, 0x96, 0xaa, 0x04, 0x8b, 0x1b, 0xbb, 0xfb, 0xad, + 0x4e, 0x9d, 0xaa, 0x08, 0xe7, 0x21, 0xbb, 0x5d, 0xdb, 0xdf, 0xae, 0xab, 0x29, 0xbc, 0x00, 0xf9, + 0x5b, 0x8d, 0x76, 0x67, 0x77, 0x9b, 0xd6, 0x9a, 0x6a, 0x1a, 0x63, 0x28, 0x71, 0xcf, 0xd8, 0x96, + 0x09, 0xb6, 0xb6, 0xf7, 0x9b, 0xcd, 0x1a, 0xfd, 0x5c, 0xcd, 0x06, 0x13, 0x60, 0xa3, 0xb5, 0xb5, + 0xab, 0xe6, 0x70, 0x11, 0xe6, 0xdb, 0x9d, 0x5a, 0xa7, 0xde, 0xae, 0x77, 0xd4, 0x39, 0xd2, 0x80, + 0x9c, 0x38, 0xfa, 0x3f, 0x67, 0x11, 0xe9, 0x42, 0x31, 0x2e, 0x39, 0xbe, 0x98, 0x50, 0x35, 0xa2, + 0xe3, 0xee, 0x98, 0x8a, 0x61, 0xfe, 0x08, 0xf9, 0x26, 0xf2, 0x27, 0xcd, 0x8d, 0x32, 0x7f, 0xbe, + 0x43, 0x50, 0x1a, 0xa7, 0xfd, 0x96, 0x61, 0xb2, 0xff, 0xa3, 0xca, 0x54, 0x60, 0xfe, 0xd0, 0x30, + 0x19, 0x8f, 0x41, 0x1c, 0x17, 0xad, 0xa7, 0xbd, 0xca, 0xb7, 0x3e, 0x86, 0x7c, 0xf4, 0x09, 0xc1, + 0x8d, 0xd4, 0x6f, 0xef, 0xd7, 0x76, 0x54, 0x25, 0xb8, 0x91, 0xd6, 0x6e, 0xa7, 0x2b, 0x96, 0x08, + 0x2f, 0x42, 0x81, 0xd6, 0xb7, 0xeb, 0x9f, 0x75, 0x9b, 0xb5, 0xce, 0x8d, 0x5b, 0x6a, 0x2a, 0xb8, + 0x22, 0x61, 0x68, 0xed, 0x4a, 0x5b, 0x7a, 0xfd, 0x8f, 0x2c, 0xcc, 0x87, 0x31, 0x06, 0x59, 0xb8, + 0xe7, 0xbb, 0xc7, 0x78, 0x65, 0xda, 0xdf, 0x84, 0xca, 0xa9, 0x09, 0xab, 0xac, 0x04, 0x0a, 0x7e, + 0x1f, 0xb2, 0x7c, 0xb2, 0xc4, 0x53, 0x27, 0xf5, 0xca, 0xf4, 0xf9, 0x9b, 0x28, 0xf8, 0x26, 0x14, + 0x62, 0x13, 0xe9, 0x8c, 0xdd, 0xe7, 0x12, 0xd6, 0xe4, 0xf0, 0x4a, 0x94, 0xab, 0x08, 0xdf, 0x82, + 0x42, 0x6c, 0x6e, 0xc3, 0x95, 0x44, 0xd2, 0x24, 0x26, 0xcb, 0x31, 0xd7, 0x94, 0x41, 0x8f, 0x28, + 0xb8, 0x0e, 0x30, 0x1e, 0xad, 0xf0, 0xd9, 0x04, 0x38, 0x3e, 0xdb, 0x55, 0x2a, 0xd3, 0x5c, 0x11, + 0xcd, 0x26, 0xe4, 0xa3, 0xc1, 0x02, 0x97, 0xa7, 0xcc, 0x1a, 0x82, 0x64, 0xf6, 0x14, 0x42, 0x14, + 0xbc, 0x05, 0xc5, 0x9a, 0x69, 0xbe, 0x0c, 0x4d, 0x25, 0xee, 0x71, 0x27, 0x79, 0xcc, 0xa8, 0xcd, + 0x4e, 0xf6, 0x72, 0x7c, 0x29, 0x59, 0x64, 0x66, 0x0d, 0x28, 0x95, 0x37, 0xff, 0x11, 0x17, 0x9d, + 0xd6, 0x81, 0xc5, 0x89, 0xa6, 0x8e, 0xb5, 0x89, 0xdd, 0x13, 0x73, 0x40, 0xa5, 0x3a, 0xd3, 0x1f, + 0xb1, 0x36, 0xa1, 0x94, 0x6c, 0x42, 0x78, 0xd6, 0xdf, 0x93, 0x4a, 0x74, 0xda, 0x8c, 0xae, 0xa5, + 0xac, 0xa2, 0xcd, 0x0f, 0x1f, 0x3d, 0xd5, 0x94, 0xc7, 0x4f, 0x35, 0xe5, 0xf9, 0x53, 0x0d, 0x7d, + 0x3b, 0xd2, 0xd0, 0x8f, 0x23, 0x0d, 0x3d, 0x1c, 0x69, 0xe8, 0xd1, 0x48, 0x43, 0xbf, 0x8d, 0x34, + 0xf4, 0xfb, 0x48, 0x53, 0x9e, 0x8f, 0x34, 0xf4, 0xe0, 0x99, 0xa6, 0x3c, 0x7a, 0xa6, 0x29, 0x8f, + 0x9f, 0x69, 0xca, 0x17, 0xb9, 0x9e, 0x69, 0x30, 0xdb, 0x3b, 0xc8, 0xf1, 0x7f, 0xee, 0xef, 0xfd, + 0x15, 0x00, 0x00, 0xff, 0xff, 0x69, 0x90, 0x7e, 0xba, 0x00, 0x10, 0x00, 0x00, } func (x MatchType) String() string { @@ -1963,6 +1996,12 @@ func (this *LabelValuesRequest) Equal(that interface{}) bool { if this.LabelName != that1.LabelName { return false } + if this.StartTimestampMs != that1.StartTimestampMs { + return false + } + if this.EndTimestampMs != that1.EndTimestampMs { + return false + } return true } func (this *LabelValuesResponse) Equal(that interface{}) bool { @@ -2013,6 +2052,12 @@ func (this *LabelNamesRequest) Equal(that interface{}) bool { } else if this == nil { return false } + if this.StartTimestampMs != that1.StartTimestampMs { + return false + } + if this.EndTimestampMs != that1.EndTimestampMs { + return false + } return true } func (this *LabelNamesResponse) Equal(that interface{}) bool { @@ -2714,9 +2759,11 @@ func (this *LabelValuesRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 7) s = append(s, "&client.LabelValuesRequest{") s = append(s, "LabelName: "+fmt.Sprintf("%#v", this.LabelName)+",\n") + s = append(s, "StartTimestampMs: "+fmt.Sprintf("%#v", this.StartTimestampMs)+",\n") + s = append(s, "EndTimestampMs: "+fmt.Sprintf("%#v", this.EndTimestampMs)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2734,8 +2781,10 @@ func (this *LabelNamesRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 4) + s := make([]string, 0, 6) s = append(s, "&client.LabelNamesRequest{") + s = append(s, "StartTimestampMs: "+fmt.Sprintf("%#v", this.StartTimestampMs)+",\n") + s = append(s, "EndTimestampMs: "+fmt.Sprintf("%#v", this.EndTimestampMs)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -3768,6 +3817,16 @@ func (m *LabelValuesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.EndTimestampMs != 0 { + i = encodeVarintCortex(dAtA, i, uint64(m.EndTimestampMs)) + i-- + dAtA[i] = 0x18 + } + if m.StartTimestampMs != 0 { + i = encodeVarintCortex(dAtA, i, uint64(m.StartTimestampMs)) + i-- + dAtA[i] = 0x10 + } if len(m.LabelName) > 0 { i -= len(m.LabelName) copy(dAtA[i:], m.LabelName) @@ -3830,6 +3889,16 @@ func (m *LabelNamesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.EndTimestampMs != 0 { + i = encodeVarintCortex(dAtA, i, uint64(m.EndTimestampMs)) + i-- + dAtA[i] = 0x10 + } + if m.StartTimestampMs != 0 { + i = encodeVarintCortex(dAtA, i, uint64(m.StartTimestampMs)) + i-- + dAtA[i] = 0x8 + } return len(dAtA) - i, nil } @@ -4769,6 +4838,12 @@ func (m *LabelValuesRequest) Size() (n int) { if l > 0 { n += 1 + l + sovCortex(uint64(l)) } + if m.StartTimestampMs != 0 { + n += 1 + sovCortex(uint64(m.StartTimestampMs)) + } + if m.EndTimestampMs != 0 { + n += 1 + sovCortex(uint64(m.EndTimestampMs)) + } return n } @@ -4793,6 +4868,12 @@ func (m *LabelNamesRequest) Size() (n int) { } var l int _ = l + if m.StartTimestampMs != 0 { + n += 1 + sovCortex(uint64(m.StartTimestampMs)) + } + if m.EndTimestampMs != 0 { + n += 1 + sovCortex(uint64(m.EndTimestampMs)) + } return n } @@ -5266,6 +5347,8 @@ func (this *LabelValuesRequest) String() string { } s := strings.Join([]string{`&LabelValuesRequest{`, `LabelName:` + fmt.Sprintf("%v", this.LabelName) + `,`, + `StartTimestampMs:` + fmt.Sprintf("%v", this.StartTimestampMs) + `,`, + `EndTimestampMs:` + fmt.Sprintf("%v", this.EndTimestampMs) + `,`, `}`, }, "") return s @@ -5285,6 +5368,8 @@ func (this *LabelNamesRequest) String() string { return "nil" } s := strings.Join([]string{`&LabelNamesRequest{`, + `StartTimestampMs:` + fmt.Sprintf("%v", this.StartTimestampMs) + `,`, + `EndTimestampMs:` + fmt.Sprintf("%v", this.EndTimestampMs) + `,`, `}`, }, "") return s @@ -6313,6 +6398,44 @@ func (m *LabelValuesRequest) Unmarshal(dAtA []byte) error { } m.LabelName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimestampMs", wireType) + } + m.StartTimestampMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCortex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTimestampMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EndTimestampMs", wireType) + } + m.EndTimestampMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCortex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EndTimestampMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipCortex(dAtA[iNdEx:]) @@ -6451,6 +6574,44 @@ func (m *LabelNamesRequest) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: LabelNamesRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimestampMs", wireType) + } + m.StartTimestampMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCortex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTimestampMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EndTimestampMs", wireType) + } + m.EndTimestampMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCortex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EndTimestampMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipCortex(dAtA[iNdEx:]) diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto b/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto index 2c3d703ce726..60e65616c626 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto @@ -63,6 +63,8 @@ message QueryStreamResponse { message LabelValuesRequest { string label_name = 1; + int64 start_timestamp_ms = 2; + int64 end_timestamp_ms = 3; } message LabelValuesResponse { @@ -70,6 +72,8 @@ message LabelValuesResponse { } message LabelNamesRequest { + int64 start_timestamp_ms = 1; + int64 end_timestamp_ms = 2; } message LabelNamesResponse { diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go index be319bc9eabc..a5087aa26b60 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/ingester_v2.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "math" "net/http" "os" "path/filepath" @@ -16,17 +17,16 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/gate" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/shipper" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" "go.uber.org/atomic" + "golang.org/x/sync/errgroup" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" @@ -48,13 +48,18 @@ type Shipper interface { } type userTSDB struct { - *tsdb.DB + db *tsdb.DB userID string refCache *cortex_tsdb.RefCache activeSeries *ActiveSeries seriesInMetric *metricCounter limiter *Limiter + forcedCompactionInProgressMtx sync.RWMutex + forcedCompactionInProgress bool + + pushesInFlight sync.WaitGroup + // Used to detect idle TSDBs. lastUpdate *atomic.Int64 @@ -66,6 +71,69 @@ type userTSDB struct { ingestedRuleSamples *ewmaRate } +// Explicitly wrapping the tsdb.DB functions that we use. + +func (u *userTSDB) Appender(ctx context.Context) storage.Appender { + return u.db.Appender(ctx) +} + +func (u *userTSDB) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return u.db.Querier(ctx, mint, maxt) +} + +func (u *userTSDB) Head() *tsdb.Head { + return u.db.Head() +} + +func (u *userTSDB) Blocks() []*tsdb.Block { + return u.db.Blocks() +} + +func (u *userTSDB) Close() error { + return u.db.Close() +} + +func (u *userTSDB) Compact() error { + return u.db.Compact() +} + +func (u *userTSDB) StartTime() (int64, error) { + return u.db.StartTime() +} + +// compactHead compacts the Head block at specified block durations avoiding a single huge block. +func (u *userTSDB) compactHead(blockDuration int64) error { + u.forcedCompactionInProgressMtx.Lock() + u.forcedCompactionInProgress = true + u.forcedCompactionInProgressMtx.Unlock() + defer func() { + u.forcedCompactionInProgressMtx.Lock() + u.forcedCompactionInProgress = false + u.forcedCompactionInProgressMtx.Unlock() + }() + // Ingestion of samples in parallel with forced compaction can lead to overlapping blocks. + // So we wait for existing in-flight requests to finish. Future push requests would fail until compaction is over. + u.pushesInFlight.Wait() + + h := u.Head() + + minTime, maxTime := h.MinTime(), h.MaxTime() + + for (minTime/blockDuration)*blockDuration != (maxTime/blockDuration)*blockDuration { + // Data in Head spans across multiple block ranges, so we break it into blocks here. + // Block max time is exclusive, so we do a -1 here. + blockMaxTime := ((minTime/blockDuration)+1)*blockDuration - 1 + if err := u.db.CompactHead(tsdb.NewRangeHead(h, minTime, blockMaxTime)); err != nil { + return err + } + + // Get current min/max times after compaction. + minTime, maxTime = h.MinTime(), h.MaxTime() + } + + return u.db.CompactHead(tsdb.NewRangeHead(h, minTime, maxTime)) +} + // PreCreation implements SeriesLifecycleCallback interface. func (u *userTSDB) PreCreation(metric labels.Labels) error { if u.limiter == nil { @@ -73,7 +141,7 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error { } // Total series limit. - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.DB.Head().NumSeries())); err != nil { + if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { return makeLimitError(perUserSeriesLimit, err) } @@ -113,15 +181,15 @@ func (u *userTSDB) PostDeletion(metrics ...labels.Labels) { // blocksToDelete filters the input blocks and returns the blocks which are safe to be deleted from the ingester. func (u *userTSDB) blocksToDelete(blocks []*tsdb.Block) map[ulid.ULID]struct{} { - if u.DB == nil { + if u.db == nil { return nil } - deletable := tsdb.DefaultBlocksToDelete(u.DB)(blocks) + deletable := tsdb.DefaultBlocksToDelete(u.db)(blocks) if u.shipper == nil { return deletable } - shipperMeta, err := shipper.ReadMetaFile(u.Dir()) + shipperMeta, err := shipper.ReadMetaFile(u.db.Dir()) if err != nil { // If there is any issue with the shipper, we should be conservative and not delete anything. level.Error(util.Logger).Log("msg", "failed to read shipper meta during deletion of blocks", "user", u.userID, "err", err) @@ -455,6 +523,11 @@ func (i *Ingester) v2Push(ctx context.Context, req *client.WriteRequest) (*clien } i.userStatesMtx.RUnlock() + if err := db.acquireAppendLock(); err != nil { + return &client.WriteResponse{}, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(err, userID).Error()) + } + defer db.releaseAppendLock() + // Given metadata is a best-effort approach, and we don't halt on errors // process it before samples. Otherwise, we risk returning an error before ingestion. i.pushMetadata(ctx, userID, req.GetMetadata()) @@ -605,6 +678,22 @@ func (i *Ingester) v2Push(ctx context.Context, req *client.WriteRequest) (*clien return &client.WriteResponse{}, nil } +func (u *userTSDB) acquireAppendLock() error { + u.forcedCompactionInProgressMtx.RLock() + defer u.forcedCompactionInProgressMtx.RUnlock() + + if u.forcedCompactionInProgress { + return errors.New("forced compaction in progress") + } + + u.pushesInFlight.Add(1) + return nil +} + +func (u *userTSDB) releaseAppendLock() { + u.pushesInFlight.Done() +} + func (i *Ingester) v2Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { userID, err := user.ExtractOrgID(ctx) if err != nil { @@ -672,10 +761,12 @@ func (i *Ingester) v2LabelValues(ctx context.Context, req *client.LabelValuesReq return &client.LabelValuesResponse{}, nil } - // Since ingester may run with a variable TSDB retention which could be few days long, - // we only query the TSDB head time range in order to avoid heavy queries (which could - // lead to ingesters out-of-memory) in case the TSDB retention is several days. - q, err := db.Querier(ctx, db.Head().MinTime(), db.Head().MaxTime()) + mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) if err != nil { return nil, err } @@ -702,10 +793,12 @@ func (i *Ingester) v2LabelNames(ctx context.Context, req *client.LabelNamesReque return &client.LabelNamesResponse{}, nil } - // Since ingester may run with a variable TSDB retention which could be few days long, - // we only query the TSDB head time range in order to avoid heavy queries (which could - // lead to ingesters out-of-memory) in case the TSDB retention is several days. - q, err := db.Querier(ctx, db.Head().MinTime(), db.Head().MaxTime()) + mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) if err != nil { return nil, err } @@ -738,10 +831,12 @@ func (i *Ingester) v2MetricsForLabelMatchers(ctx context.Context, req *client.Me return nil, err } - // Since ingester may run with a variable TSDB retention which could be few days long, - // we only query the TSDB head time range in order to avoid heavy queries (which could - // lead to ingesters out-of-memory) in case the TSDB retention is several days. - q, err := db.Querier(ctx, db.Head().MinTime(), db.Head().MaxTime()) + mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) if err != nil { return nil, err } @@ -1005,6 +1100,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { NoLockfile: true, StripeSize: i.cfg.BlocksStorageConfig.TSDB.StripeSize, WALCompression: i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled, + WALSegmentSize: i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes, SeriesLifecycleCallback: userDB, BlocksToDelete: userDB.blocksToDelete, }) @@ -1022,7 +1118,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { return nil, errors.Wrapf(err, "failed to compact TSDB: %s", udir) } - userDB.DB = db + userDB.db = db // We set the limiter here because we don't want to limit // series during WAL replay. userDB.limiter = i.limiter @@ -1096,98 +1192,100 @@ func (i *Ingester) closeAllTSDB() { // concurrently opening TSDB. func (i *Ingester) openExistingTSDB(ctx context.Context) error { level.Info(util.Logger).Log("msg", "opening existing TSDBs") - wg := &sync.WaitGroup{} - openGate := gate.New(i.cfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup) - // Keep track of all errors that could occur. - errs := tsdb_errors.MultiError{} - errsMx := sync.Mutex{} + queue := make(chan string) + group, groupCtx := errgroup.WithContext(ctx) - walkErr := filepath.Walk(i.cfg.BlocksStorageConfig.TSDB.Dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - // If the root directory doesn't exist, we're OK (not needed to be created upfront). - if os.IsNotExist(err) && path == i.cfg.BlocksStorageConfig.TSDB.Dir { - return filepath.SkipDir - } + // Create a pool of workers which will open existing TSDBs. + for n := 0; n < i.cfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup; n++ { + group.Go(func() error { + for userID := range queue { + startTime := time.Now() - level.Error(util.Logger).Log("msg", "an error occurred while iterating the filesystem storing TSDBs", "path", path, "err", err) - return errors.Wrapf(err, "an error occurred while iterating the filesystem storing TSDBs at %s", path) - } + db, err := i.createTSDB(userID) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to open TSDB", "err", err, "user", userID) + return errors.Wrapf(err, "unable to open TSDB for user %s", userID) + } + + // Add the database to the map of user databases + i.userStatesMtx.Lock() + i.TSDBState.dbs[userID] = db + i.userStatesMtx.Unlock() + i.metrics.memUsers.Inc() + + i.TSDBState.walReplayTime.Observe(time.Since(startTime).Seconds()) + } - // Skip root dir and all other files - if path == i.cfg.BlocksStorageConfig.TSDB.Dir || !info.IsDir() { return nil - } + }) + } - // Top level directories are assumed to be user TSDBs - userID := info.Name() - f, err := os.Open(path) - if err != nil { - level.Error(util.Logger).Log("msg", "unable to open TSDB dir", "err", err, "user", userID, "path", path) - return errors.Wrapf(err, "unable to open TSDB dir %s for user %s", path, userID) - } - defer f.Close() + // Spawn a goroutine to find all users with a TSDB on the filesystem. + group.Go(func() error { + // Close the queue once filesystem walking is done. + defer close(queue) + + walkErr := filepath.Walk(i.cfg.BlocksStorageConfig.TSDB.Dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If the root directory doesn't exist, we're OK (not needed to be created upfront). + if os.IsNotExist(err) && path == i.cfg.BlocksStorageConfig.TSDB.Dir { + return filepath.SkipDir + } - // If the dir is empty skip it - if _, err := f.Readdirnames(1); err != nil { - if err == io.EOF { - return filepath.SkipDir + level.Error(util.Logger).Log("msg", "an error occurred while iterating the filesystem storing TSDBs", "path", path, "err", err) + return errors.Wrapf(err, "an error occurred while iterating the filesystem storing TSDBs at %s", path) } - level.Error(util.Logger).Log("msg", "unable to read TSDB dir", "err", err, "user", userID, "path", path) - return errors.Wrapf(err, "unable to read TSDB dir %s for user %s", path, userID) - } + // Skip root dir and all other files + if path == i.cfg.BlocksStorageConfig.TSDB.Dir || !info.IsDir() { + return nil + } - // Limit the number of TSDB's opening concurrently. Start blocks until there's a free spot available or the context is cancelled. - if err := openGate.Start(ctx); err != nil { - return err - } + // Top level directories are assumed to be user TSDBs + userID := info.Name() + f, err := os.Open(path) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to open TSDB dir", "err", err, "user", userID, "path", path) + return errors.Wrapf(err, "unable to open TSDB dir %s for user %s", path, userID) + } + defer f.Close() - wg.Add(1) - go func(userID string) { - defer wg.Done() - defer openGate.Done() - defer func(ts time.Time) { - i.TSDBState.walReplayTime.Observe(time.Since(ts).Seconds()) - }(time.Now()) + // If the dir is empty skip it + if _, err := f.Readdirnames(1); err != nil { + if err == io.EOF { + return filepath.SkipDir + } - db, err := i.createTSDB(userID) - if err != nil { - errsMx.Lock() - errs.Add(errors.Wrapf(err, "unable to open TSDB for user %s", userID)) - errsMx.Unlock() + level.Error(util.Logger).Log("msg", "unable to read TSDB dir", "err", err, "user", userID, "path", path) + return errors.Wrapf(err, "unable to read TSDB dir %s for user %s", path, userID) + } - level.Error(util.Logger).Log("msg", "unable to open TSDB", "err", err, "user", userID) - return + // Enqueue the user to be processed. + select { + case queue <- userID: + // Nothing to do. + case <-groupCtx.Done(): + // Interrupt in case a failure occurred in another goroutine. + return nil } - // Add the database to the map of user databases - i.userStatesMtx.Lock() - i.TSDBState.dbs[userID] = db - i.userStatesMtx.Unlock() - i.metrics.memUsers.Inc() - }(userID) + // Don't descend into subdirectories. + return filepath.SkipDir + }) - return filepath.SkipDir // Don't descend into directories + return errors.Wrapf(walkErr, "unable to walk directory %s containing existing TSDBs", i.cfg.BlocksStorageConfig.TSDB.Dir) }) - if walkErr != nil { - errsMx.Lock() - errs.Add(errors.Wrapf(walkErr, "unable to walk directory %s containing existing TSDBs", i.cfg.BlocksStorageConfig.TSDB.Dir)) - errsMx.Unlock() - } - - // Wait for all opening routines to finish - wg.Wait() - - // Ensure no error occurred. - if errs.Err() == nil { - level.Info(util.Logger).Log("msg", "successfully opened existing TSDBs") - return nil + // Wait for all workers to complete. + err := group.Wait() + if err != nil { + level.Error(util.Logger).Log("msg", "error while opening existing TSDBs", "err", err) + return err } - level.Error(util.Logger).Log("msg", "error while opening existing TSDBs", "err", errs.Error()) - return errs.Err() + level.Info(util.Logger).Log("msg", "successfully opened existing TSDBs") + return nil } // numSeriesInTSDB returns the total number of in-memory series across all open TSDBs. @@ -1313,12 +1411,12 @@ func (i *Ingester) compactBlocks(ctx context.Context, force bool) { switch { case force: reason = "forced" - err = userDB.CompactHead(tsdb.NewRangeHead(h, h.MinTime(), h.MaxTime())) + err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) case i.cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout > 0 && userDB.isIdle(time.Now(), i.cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout): reason = "idle" level.Info(util.Logger).Log("msg", "TSDB is idle, forcing compaction", "user", userID) - err = userDB.CompactHead(tsdb.NewRangeHead(h, h.MinTime(), h.MaxTime())) + err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) default: reason = "regular" @@ -1440,3 +1538,30 @@ func (i *Ingester) v2FlushHandler(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) } + +// metadataQueryRange returns the best range to query for metadata queries based on the timerange in the ingester. +func metadataQueryRange(queryStart, queryEnd int64, db *userTSDB) (mint, maxt int64, err error) { + // Ingesters are run with limited retention and we don't support querying the store-gateway for labels yet. + // This means if someone loads a dashboard that is outside the range of the ingester, and we only return the + // data for the timerange requested (which will be empty), the dashboards will break. To fix this we should + // return the "head block" range until we can query the store-gateway. + + // Now the question would be what to do when the query is partially in the ingester range. I would err on the side + // of caution and query the entire db, as I can't think of a good way to query the head + the overlapping range. + mint, maxt = queryStart, queryEnd + + lowestTs, err := db.StartTime() + if err != nil { + return mint, maxt, err + } + + // Completely outside. + if queryEnd < lowestTs { + mint, maxt = db.Head().MinTime(), db.Head().MaxTime() + } else if queryStart < lowestTs { + // Partially inside. + mint, maxt = 0, math.MaxInt64 + } + + return +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/metrics.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/metrics.go index 54fa26924516..6a0d31582af1 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/metrics.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/metrics.go @@ -232,10 +232,14 @@ type tsdbMetrics struct { tsdbFsyncDuration *prometheus.Desc tsdbPageFlushes *prometheus.Desc tsdbPageCompletions *prometheus.Desc - tsdbTruncateFail *prometheus.Desc - tsdbTruncateTotal *prometheus.Desc - tsdbTruncateDuration *prometheus.Desc - tsdbWritesFailed *prometheus.Desc + tsdbWALTruncateFail *prometheus.Desc + tsdbWALTruncateTotal *prometheus.Desc + tsdbWALTruncateDuration *prometheus.Desc + tsdbWALCorruptionsTotal *prometheus.Desc + tsdbWALWritesFailed *prometheus.Desc + tsdbHeadTruncateFail *prometheus.Desc + tsdbHeadTruncateTotal *prometheus.Desc + tsdbHeadGcDuration *prometheus.Desc tsdbActiveAppenders *prometheus.Desc tsdbSeriesNotFound *prometheus.Desc tsdbChunks *prometheus.Desc @@ -296,22 +300,38 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics { "cortex_ingester_tsdb_wal_completed_pages_total", "Total number of TSDB WAL completed pages.", nil, nil), - tsdbTruncateFail: prometheus.NewDesc( + tsdbWALTruncateFail: prometheus.NewDesc( "cortex_ingester_tsdb_wal_truncations_failed_total", "Total number of TSDB WAL truncations that failed.", nil, nil), - tsdbTruncateTotal: prometheus.NewDesc( + tsdbWALTruncateTotal: prometheus.NewDesc( "cortex_ingester_tsdb_wal_truncations_total", "Total number of TSDB WAL truncations attempted.", nil, nil), - tsdbTruncateDuration: prometheus.NewDesc( + tsdbWALTruncateDuration: prometheus.NewDesc( "cortex_ingester_tsdb_wal_truncate_duration_seconds", "Duration of TSDB WAL truncation.", nil, nil), - tsdbWritesFailed: prometheus.NewDesc( + tsdbWALCorruptionsTotal: prometheus.NewDesc( + "cortex_ingester_tsdb_wal_corruptions_total", + "Total number of TSDB WAL corruptions.", + nil, nil), + tsdbWALWritesFailed: prometheus.NewDesc( "cortex_ingester_tsdb_wal_writes_failed_total", "Total number of TSDB WAL writes that failed.", nil, nil), + tsdbHeadTruncateFail: prometheus.NewDesc( + "cortex_ingester_tsdb_head_truncations_failed_total", + "Total number of TSDB head truncations that failed.", + nil, nil), + tsdbHeadTruncateTotal: prometheus.NewDesc( + "cortex_ingester_tsdb_head_truncations_total", + "Total number of TSDB head truncations attempted.", + nil, nil), + tsdbHeadGcDuration: prometheus.NewDesc( + "cortex_ingester_tsdb_head_gc_duration_seconds", + "Runtime of garbage collection in the TSDB head.", + nil, nil), tsdbActiveAppenders: prometheus.NewDesc( "cortex_ingester_tsdb_head_active_appenders", "Number of currently active TSDB appender transactions.", @@ -374,10 +394,14 @@ func (sm *tsdbMetrics) Describe(out chan<- *prometheus.Desc) { out <- sm.tsdbFsyncDuration out <- sm.tsdbPageFlushes out <- sm.tsdbPageCompletions - out <- sm.tsdbTruncateFail - out <- sm.tsdbTruncateTotal - out <- sm.tsdbTruncateDuration - out <- sm.tsdbWritesFailed + out <- sm.tsdbWALTruncateFail + out <- sm.tsdbWALTruncateTotal + out <- sm.tsdbWALTruncateDuration + out <- sm.tsdbWALCorruptionsTotal + out <- sm.tsdbWALWritesFailed + out <- sm.tsdbHeadTruncateFail + out <- sm.tsdbHeadTruncateTotal + out <- sm.tsdbHeadGcDuration out <- sm.tsdbActiveAppenders out <- sm.tsdbSeriesNotFound out <- sm.tsdbChunks @@ -408,10 +432,14 @@ func (sm *tsdbMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfSummaries(out, sm.tsdbFsyncDuration, "prometheus_tsdb_wal_fsync_duration_seconds") data.SendSumOfCounters(out, sm.tsdbPageFlushes, "prometheus_tsdb_wal_page_flushes_total") data.SendSumOfCounters(out, sm.tsdbPageCompletions, "prometheus_tsdb_wal_completed_pages_total") - data.SendSumOfCounters(out, sm.tsdbTruncateFail, "prometheus_tsdb_wal_truncations_failed_total") - data.SendSumOfCounters(out, sm.tsdbTruncateTotal, "prometheus_tsdb_wal_truncations_total") - data.SendSumOfSummaries(out, sm.tsdbTruncateDuration, "prometheus_tsdb_wal_truncate_duration_seconds") - data.SendSumOfCounters(out, sm.tsdbWritesFailed, "prometheus_tsdb_wal_writes_failed_total") + data.SendSumOfCounters(out, sm.tsdbWALTruncateFail, "prometheus_tsdb_wal_truncations_failed_total") + data.SendSumOfCounters(out, sm.tsdbWALTruncateTotal, "prometheus_tsdb_wal_truncations_total") + data.SendSumOfSummaries(out, sm.tsdbWALTruncateDuration, "prometheus_tsdb_wal_truncate_duration_seconds") + data.SendSumOfCounters(out, sm.tsdbWALCorruptionsTotal, "prometheus_tsdb_wal_corruptions_total") + data.SendSumOfCounters(out, sm.tsdbWALWritesFailed, "prometheus_tsdb_wal_writes_failed_total") + data.SendSumOfCounters(out, sm.tsdbHeadTruncateFail, "prometheus_tsdb_head_truncations_failed_total") + data.SendSumOfCounters(out, sm.tsdbHeadTruncateTotal, "prometheus_tsdb_head_truncations_total") + data.SendSumOfSummaries(out, sm.tsdbHeadGcDuration, "prometheus_tsdb_head_gc_duration_seconds") data.SendSumOfGauges(out, sm.tsdbActiveAppenders, "prometheus_tsdb_head_active_appenders") data.SendSumOfCounters(out, sm.tsdbSeriesNotFound, "prometheus_tsdb_head_series_not_found_total") data.SendSumOfGauges(out, sm.tsdbChunks, "prometheus_tsdb_head_chunks") diff --git a/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go b/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go index 244c763d28fa..8ec6b6abd9f5 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ingester/wal.go @@ -445,7 +445,7 @@ func (w *walWrapper) deleteCheckpoints(maxIndex int) (err error) { } }() - var errs tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() files, err := ioutil.ReadDir(w.wal.Dir()) if err != nil { @@ -795,11 +795,8 @@ func processWALWithRepair(startSegment int, userStates *userStates, params walRe if err != nil { level.Error(util.Logger).Log("msg", "error in repairing WAL", "err", err) } - var multiErr tsdb_errors.MultiError - multiErr.Add(err) - multiErr.Add(w.Close()) - return multiErr.Err() + return tsdb_errors.NewMulti(err, w.Close()).Err() } // processWAL processes the records in the WAL concurrently. diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_scanner.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_scanner.go index 437dec2d55a1..8aaa99297c83 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_scanner.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_scanner.go @@ -176,7 +176,7 @@ func (d *BlocksScanner) scanBucket(ctx context.Context) (returnErr error) { resMetas := map[string][]*BlockMeta{} resMetasLookup := map[string]map[ulid.ULID]*BlockMeta{} resDeletionMarks := map[string]map[ulid.ULID]*metadata.DeletionMark{} - resErrs := tsdb_errors.MultiError{} + resErrs := tsdb_errors.NewMulti() // Create a pool of workers which will synchronize metas. The pool size // is limited in order to avoid to concurrently sync a lot of tenants in diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go index 22ebaa285797..29db5ee3e83f 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/blocks_store_queryable.go @@ -382,7 +382,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* // Fetch series from stores. If an error occur we do not retry because retries // are only meant to cover missing blocks. - seriesSets, queriedBlocks, warnings, numChunks, err := q.fetchSeriesFromStores(spanCtx, clients, minT, maxT, matchers, convertedMatchers, maxChunksLimit, leftChunksLimit) + seriesSets, queriedBlocks, warnings, numChunks, err := q.fetchSeriesFromStores(spanCtx, sp, clients, minT, maxT, matchers, convertedMatchers, maxChunksLimit, leftChunksLimit) if err != nil { return storage.ErrSeriesSet(err) } @@ -433,6 +433,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* func (q *blocksStoreQuerier) fetchSeriesFromStores( ctx context.Context, + sp *storage.SelectHints, clients map[BlocksStoreClient][]ulid.ULID, minT int64, maxT int64, @@ -459,7 +460,13 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( blockIDs := blockIDs g.Go(func() error { - req, err := createSeriesRequest(minT, maxT, convertedMatchers, blockIDs) + // See: https://github.com/prometheus/prometheus/pull/8050 + // TODO(goutham): we should ideally be passing the hints down to the storage layer + // and let the TSDB return us data with no chunks as in prometheus#8050. + // But this is an acceptable workaround for now. + skipChunks := sp != nil && sp.Func == "series" + + req, err := createSeriesRequest(minT, maxT, convertedMatchers, skipChunks, blockIDs) if err != nil { return errors.Wrapf(err, "failed to create series request") } @@ -546,7 +553,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( return seriesSets, queriedBlocks, warnings, int(numChunks.Load()), nil } -func createSeriesRequest(minT, maxT int64, matchers []storepb.LabelMatcher, blockIDs []ulid.ULID) (*storepb.SeriesRequest, error) { +func createSeriesRequest(minT, maxT int64, matchers []storepb.LabelMatcher, skipChunks bool, blockIDs []ulid.ULID) (*storepb.SeriesRequest, error) { // Selectively query only specific blocks. hints := &hintspb.SeriesRequestHints{ BlockMatchers: []storepb.LabelMatcher{ @@ -569,6 +576,7 @@ func createSeriesRequest(minT, maxT int64, matchers []storepb.LabelMatcher, bloc Matchers: matchers, PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT, Hints: anyHints, + SkipChunks: skipChunks, }, nil } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go b/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go index 2f3743543a15..4e08b02d21a7 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/distributor_queryable.go @@ -25,8 +25,8 @@ import ( type Distributor interface { Query(ctx context.Context, from, to model.Time, matchers ...*labels.Matcher) (model.Matrix, error) QueryStream(ctx context.Context, from, to model.Time, matchers ...*labels.Matcher) (*client.QueryStreamResponse, error) - LabelValuesForLabelName(context.Context, model.LabelName) ([]string, error) - LabelNames(context.Context) ([]string, error) + LabelValuesForLabelName(ctx context.Context, from, to model.Time, label model.LabelName) ([]string, error) + LabelNames(context.Context, model.Time, model.Time) ([]string, error) MetricsForLabelMatchers(ctx context.Context, from, through model.Time, matchers ...*labels.Matcher) ([]metric.Metric, error) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) } @@ -183,12 +183,12 @@ func (q *distributorQuerier) streamingSelect(minT, maxT int64, matchers []*label } func (q *distributorQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { - lv, err := q.distributor.LabelValuesForLabelName(q.ctx, model.LabelName(name)) + lv, err := q.distributor.LabelValuesForLabelName(q.ctx, model.Time(q.mint), model.Time(q.maxt), model.LabelName(name)) return lv, nil, err } func (q *distributorQuerier) LabelNames() ([]string, storage.Warnings, error) { - ln, err := q.distributor.LabelNames(q.ctx) + ln, err := q.distributor.LabelNames(q.ctx, model.Time(q.mint), model.Time(q.maxt)) return ln, nil, err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go b/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go deleted file mode 100644 index 54356592e209..000000000000 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend.go +++ /dev/null @@ -1,531 +0,0 @@ -package frontend - -import ( - "bytes" - "context" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "path" - "strings" - "sync" - "time" - - "github.com/NYTimes/gziphandler" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - opentracing "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/httpgrpc/server" - "github.com/weaveworks/common/user" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/util" -) - -const ( - // StatusClientClosedRequest is the status code for when a client request cancellation of an http request - StatusClientClosedRequest = 499 -) - -var ( - errTooManyRequest = httpgrpc.Errorf(http.StatusTooManyRequests, "too many outstanding requests") - errCanceled = httpgrpc.Errorf(StatusClientClosedRequest, context.Canceled.Error()) - errDeadlineExceeded = httpgrpc.Errorf(http.StatusGatewayTimeout, context.DeadlineExceeded.Error()) -) - -// Config for a Frontend. -type Config struct { - MaxOutstandingPerTenant int `yaml:"max_outstanding_per_tenant"` - CompressResponses bool `yaml:"compress_responses"` - DownstreamURL string `yaml:"downstream_url"` - LogQueriesLongerThan time.Duration `yaml:"log_queries_longer_than"` -} - -// RegisterFlags adds the flags required to config this to the given FlagSet. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.IntVar(&cfg.MaxOutstandingPerTenant, "querier.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per frontend; requests beyond this error with HTTP 429.") - f.BoolVar(&cfg.CompressResponses, "querier.compress-http-responses", false, "Compress HTTP responses.") - f.StringVar(&cfg.DownstreamURL, "frontend.downstream-url", "", "URL of downstream Prometheus.") - f.DurationVar(&cfg.LogQueriesLongerThan, "frontend.log-queries-longer-than", 0, "Log queries that are slower than the specified duration. Set to 0 to disable. Set to < 0 to enable on all queries.") -} - -type Limits interface { - // Returns max queriers to use per tenant, or 0 if shuffle sharding is disabled. - MaxQueriersPerUser(user string) int -} - -// Frontend queues HTTP requests, dispatches them to backends, and handles retries -// for requests which failed. -type Frontend struct { - cfg Config - log log.Logger - roundTripper http.RoundTripper - limits Limits - - mtx sync.Mutex - cond *sync.Cond // Notified when request is enqueued or dequeued, or querier is disconnected. - queues *queues - - connectedClients *atomic.Int32 - - // Metrics. - numClients prometheus.GaugeFunc - queueDuration prometheus.Histogram - queueLength *prometheus.GaugeVec -} - -type request struct { - enqueueTime time.Time - queueSpan opentracing.Span - originalCtx context.Context - - request *httpgrpc.HTTPRequest - err chan error - response chan *httpgrpc.HTTPResponse -} - -// New creates a new frontend. -func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer) (*Frontend, error) { - connectedClients := atomic.NewInt32(0) - f := &Frontend{ - cfg: cfg, - log: log, - limits: limits, - queues: newUserQueues(cfg.MaxOutstandingPerTenant), - queueDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "query_frontend_queue_duration_seconds", - Help: "Time spend by requests queued.", - Buckets: prometheus.DefBuckets, - }), - queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "query_frontend_queue_length", - Help: "Number of queries in the queue.", - }, []string{"user"}), - numClients: promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "query_frontend_connected_clients", - Help: "Number of worker clients currently connected to the frontend.", - }, func() float64 { return float64(connectedClients.Load()) }), - connectedClients: connectedClients, - } - f.cond = sync.NewCond(&f.mtx) - - // The front end implements http.RoundTripper using a GRPC worker queue by default. - f.roundTripper = f - // However if the user has specified a downstream Prometheus, then we should use that. - if cfg.DownstreamURL != "" { - u, err := url.Parse(cfg.DownstreamURL) - if err != nil { - return nil, err - } - - f.roundTripper = RoundTripFunc(func(r *http.Request) (*http.Response, error) { - tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(r.Context()) - if tracer != nil && span != nil { - carrier := opentracing.HTTPHeadersCarrier(r.Header) - tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier) - } - r.URL.Scheme = u.Scheme - r.URL.Host = u.Host - r.URL.Path = path.Join(u.Path, r.URL.Path) - r.Host = "" - return http.DefaultTransport.RoundTrip(r) - }) - } - - return f, nil -} - -// Wrap uses a Tripperware to chain a new RoundTripper to the frontend. -func (f *Frontend) Wrap(trw Tripperware) { - f.roundTripper = trw(f.roundTripper) -} - -// Tripperware is a signature for all http client-side middleware. -type Tripperware func(http.RoundTripper) http.RoundTripper - -// RoundTripFunc is to http.RoundTripper what http.HandlerFunc is to http.Handler. -type RoundTripFunc func(*http.Request) (*http.Response, error) - -// RoundTrip implements http.RoundTripper. -func (f RoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { - return f(r) -} - -// Close stops new requests and errors out any pending requests. -func (f *Frontend) Close() { - f.mtx.Lock() - defer f.mtx.Unlock() - for f.queues.len() > 0 { - f.cond.Wait() - } -} - -// Handler for HTTP requests. -func (f *Frontend) Handler() http.Handler { - if f.cfg.CompressResponses { - return gziphandler.GzipHandler(http.HandlerFunc(f.handle)) - } - return http.HandlerFunc(f.handle) -} - -func (f *Frontend) handle(w http.ResponseWriter, r *http.Request) { - - startTime := time.Now() - resp, err := f.roundTripper.RoundTrip(r) - queryResponseTime := time.Since(startTime) - - if err != nil { - writeError(w, err) - } else { - hs := w.Header() - for h, vs := range resp.Header { - hs[h] = vs - } - w.WriteHeader(resp.StatusCode) - io.Copy(w, resp.Body) - } - - // If LogQueriesLongerThan is set to <0 we log every query, if it is set to 0 query logging - // is disabled - if f.cfg.LogQueriesLongerThan != 0 && queryResponseTime > f.cfg.LogQueriesLongerThan { - logMessage := []interface{}{ - "msg", "slow query detected", - "method", r.Method, - "host", r.Host, - "path", r.URL.Path, - "time_taken", queryResponseTime.String(), - } - - // Ensure the form has been parsed so all the parameters are present - err = r.ParseForm() - if err != nil { - level.Warn(util.WithContext(r.Context(), f.log)).Log("msg", "unable to parse form for request", "err", err) - } - - // Attempt to iterate through the Form to log any filled in values - for k, v := range r.Form { - logMessage = append(logMessage, fmt.Sprintf("param_%s", k), strings.Join(v, ",")) - } - - level.Info(util.WithContext(r.Context(), f.log)).Log(logMessage...) - } -} - -func writeError(w http.ResponseWriter, err error) { - switch err { - case context.Canceled: - err = errCanceled - case context.DeadlineExceeded: - err = errDeadlineExceeded - default: - } - server.WriteError(w, err) -} - -// RoundTrip implement http.Transport. -func (f *Frontend) RoundTrip(r *http.Request) (*http.Response, error) { - req, err := server.HTTPRequest(r) - if err != nil { - return nil, err - } - - resp, err := f.RoundTripGRPC(r.Context(), req) - if err != nil { - return nil, err - } - - httpResp := &http.Response{ - StatusCode: int(resp.Code), - Body: ioutil.NopCloser(bytes.NewReader(resp.Body)), - Header: http.Header{}, - } - for _, h := range resp.Headers { - httpResp.Header[h.Key] = h.Values - } - return httpResp, nil -} - -type httpgrpcHeadersCarrier httpgrpc.HTTPRequest - -func (c *httpgrpcHeadersCarrier) Set(key, val string) { - c.Headers = append(c.Headers, &httpgrpc.Header{ - Key: key, - Values: []string{val}, - }) -} - -// RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { - // Propagate trace context in gRPC too - this will be ignored if using HTTP. - tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) - if tracer != nil && span != nil { - carrier := (*httpgrpcHeadersCarrier)(req) - tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier) - } - - request := request{ - request: req, - originalCtx: ctx, - - // Buffer of 1 to ensure response can be written by the server side - // of the Process stream, even if this goroutine goes away due to - // client context cancellation. - err: make(chan error, 1), - response: make(chan *httpgrpc.HTTPResponse, 1), - } - - if err := f.queueRequest(ctx, &request); err != nil { - return nil, err - } - - select { - case <-ctx.Done(): - return nil, ctx.Err() - - case resp := <-request.response: - return resp, nil - - case err := <-request.err: - return nil, err - } -} - -// Process allows backends to pull requests from the frontend. -func (f *Frontend) Process(server Frontend_ProcessServer) error { - querierID, err := getQuerierID(server) - if err != nil { - return err - } - - f.registerQuerierConnection(querierID) - defer f.unregisterQuerierConnection(querierID) - - // If the downstream request(from querier -> frontend) is cancelled, - // we need to ping the condition variable to unblock getNextRequestForQuerier. - // Ideally we'd have ctx aware condition variables... - go func() { - <-server.Context().Done() - f.cond.Broadcast() - }() - - lastUserIndex := -1 - - for { - req, idx, err := f.getNextRequestForQuerier(server.Context(), lastUserIndex, querierID) - if err != nil { - return err - } - lastUserIndex = idx - - // Handle the stream sending & receiving on a goroutine so we can - // monitoring the contexts in a select and cancel things appropriately. - resps := make(chan *httpgrpc.HTTPResponse, 1) - errs := make(chan error, 1) - go func() { - err = server.Send(&FrontendToClient{ - Type: HTTP_REQUEST, - HttpRequest: req.request, - }) - if err != nil { - errs <- err - return - } - - resp, err := server.Recv() - if err != nil { - errs <- err - return - } - - resps <- resp.HttpResponse - }() - - select { - // If the upstream request is cancelled, we need to cancel the - // downstream req. Only way we can do that is to close the stream. - // The worker client is expecting this semantics. - case <-req.originalCtx.Done(): - return req.originalCtx.Err() - - // Is there was an error handling this request due to network IO, - // then error out this upstream request _and_ stream. - case err := <-errs: - req.err <- err - return err - - // Happy path: propagate the response. - case resp := <-resps: - req.response <- resp - } - } -} - -func getQuerierID(server Frontend_ProcessServer) (string, error) { - err := server.Send(&FrontendToClient{ - Type: GET_ID, - // Old queriers don't support GET_ID, and will try to use the request. - // To avoid confusing them, include dummy request. - HttpRequest: &httpgrpc.HTTPRequest{ - Method: "GET", - Url: "/invalid_request_sent_by_frontend", - }, - }) - - if err != nil { - return "", err - } - - resp, err := server.Recv() - - // Old queriers will return empty string, which is fine. All old queriers will be - // treated as single querier with lot of connections. - // (Note: if resp is nil, GetClientID() returns "") - return resp.GetClientID(), err -} - -func (f *Frontend) queueRequest(ctx context.Context, req *request) error { - userID, err := user.ExtractOrgID(ctx) - if err != nil { - return err - } - - req.enqueueTime = time.Now() - req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") - - maxQueriers := f.limits.MaxQueriersPerUser(userID) - - f.mtx.Lock() - defer f.mtx.Unlock() - - queue := f.queues.getOrAddQueue(userID, maxQueriers) - if queue == nil { - // This can only happen if userID is "". - return errors.New("no queue found") - } - - select { - case queue <- req: - f.queueLength.WithLabelValues(userID).Inc() - f.cond.Broadcast() - return nil - default: - return errTooManyRequest - } -} - -// getQueue picks a random queue and takes the next unexpired request off of it, so we -// fairly process users queries. Will block if there are no requests. -func (f *Frontend) getNextRequestForQuerier(ctx context.Context, lastUserIndex int, querierID string) (*request, int, error) { - f.mtx.Lock() - defer f.mtx.Unlock() - - querierWait := false - -FindQueue: - // We need to wait if there are no users, or no pending requests for given querier. - for (f.queues.len() == 0 || querierWait) && ctx.Err() == nil { - querierWait = false - f.cond.Wait() - } - - if err := ctx.Err(); err != nil { - return nil, lastUserIndex, err - } - - for { - queue, userID, idx := f.queues.getNextQueueForQuerier(lastUserIndex, querierID) - lastUserIndex = idx - if queue == nil { - break - } - /* - We want to dequeue the next unexpired request from the chosen tenant queue. - The chance of choosing a particular tenant for dequeueing is (1/active_tenants). - This is problematic under load, especially with other middleware enabled such as - querier.split-by-interval, where one request may fan out into many. - If expired requests aren't exhausted before checking another tenant, it would take - n_active_tenants * n_expired_requests_at_front_of_queue requests being processed - before an active request was handled for the tenant in question. - If this tenant meanwhile continued to queue requests, - it's possible that it's own queue would perpetually contain only expired requests. - */ - - // Pick the first non-expired request from this user's queue (if any). - for { - lastRequest := false - request := <-queue - if len(queue) == 0 { - f.queues.deleteQueue(userID) - lastRequest = true - } - - // Tell close() we've processed a request. - f.cond.Broadcast() - - f.queueDuration.Observe(time.Since(request.enqueueTime).Seconds()) - f.queueLength.WithLabelValues(userID).Dec() - request.queueSpan.Finish() - - // Ensure the request has not already expired. - if request.originalCtx.Err() == nil { - return request, lastUserIndex, nil - } - - // Stop iterating on this queue if we've just consumed the last request. - if lastRequest { - break - } - } - } - - // There are no unexpired requests, so we can get back - // and wait for more requests. - querierWait = true - goto FindQueue -} - -// CheckReady determines if the query frontend is ready. Function parameters/return -// chosen to match the same method in the ingester -func (f *Frontend) CheckReady(_ context.Context) error { - // if the downstream url is configured the query frontend is not aware of the state - // of the queriers and is therefore always ready - if f.cfg.DownstreamURL != "" { - return nil - } - - // if we have more than one querier connected we will consider ourselves ready - connectedClients := f.connectedClients.Load() - if connectedClients > 0 { - return nil - } - - msg := fmt.Sprintf("not ready: number of queriers connected to query-frontend is %d", connectedClients) - level.Info(f.log).Log("msg", msg) - return errors.New(msg) -} - -func (f *Frontend) registerQuerierConnection(querier string) { - f.connectedClients.Inc() - - f.mtx.Lock() - defer f.mtx.Unlock() - f.queues.addQuerierConnection(querier) -} - -func (f *Frontend) unregisterQuerierConnection(querier string) { - f.connectedClients.Dec() - - f.mtx.Lock() - defer f.mtx.Unlock() - f.queues.removeQuerierConnection(querier) -} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker.go b/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker.go deleted file mode 100644 index 48f133659cdd..000000000000 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker.go +++ /dev/null @@ -1,219 +0,0 @@ -package frontend - -import ( - "context" - "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" - "github.com/weaveworks/common/httpgrpc/server" - "github.com/weaveworks/common/middleware" - "google.golang.org/grpc" - "google.golang.org/grpc/naming" - - "github.com/cortexproject/cortex/pkg/querier" - "github.com/cortexproject/cortex/pkg/util/grpcclient" - "github.com/cortexproject/cortex/pkg/util/services" -) - -// WorkerConfig is config for a worker. -type WorkerConfig struct { - Address string `yaml:"frontend_address"` - Parallelism int `yaml:"parallelism"` - MatchMaxConcurrency bool `yaml:"match_max_concurrent"` - DNSLookupDuration time.Duration `yaml:"dns_lookup_duration"` - QuerierID string `yaml:"id"` - - GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` -} - -// RegisterFlags adds the flags required to config this to the given FlagSet. -func (cfg *WorkerConfig) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.Address, "querier.frontend-address", "", "Address of query frontend service, in host:port format.") - f.IntVar(&cfg.Parallelism, "querier.worker-parallelism", 10, "Number of simultaneous queries to process per query frontend.") - f.BoolVar(&cfg.MatchMaxConcurrency, "querier.worker-match-max-concurrent", false, "Force worker concurrency to match the -querier.max-concurrent option. Overrides querier.worker-parallelism.") - f.DurationVar(&cfg.DNSLookupDuration, "querier.dns-lookup-period", 10*time.Second, "How often to query DNS.") - f.StringVar(&cfg.QuerierID, "querier.id", "", "Querier ID, sent to frontend service to identify requests from the same querier. Defaults to hostname.") - - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("querier.frontend-client", f) -} - -func (cfg *WorkerConfig) Validate(log log.Logger) error { - return cfg.GRPCClientConfig.Validate(log) -} - -// Worker is the counter-part to the frontend, actually processing requests. -type worker struct { - cfg WorkerConfig - querierCfg querier.Config - log log.Logger - server *server.Server - - watcher naming.Watcher //nolint:staticcheck //Skipping for now. If you still see this more than likely issue https://github.com/cortexproject/cortex/issues/2015 has not yet been addressed. - managers map[string]*frontendManager -} - -// NewWorker creates a new worker and returns a service that is wrapping it. -// If no address is specified, it returns nil service (and no error). -func NewWorker(cfg WorkerConfig, querierCfg querier.Config, server *server.Server, log log.Logger) (services.Service, error) { - if cfg.Address == "" { - level.Info(log).Log("msg", "no address specified, not starting worker") - return nil, nil - } - - if cfg.QuerierID == "" { - hostname, err := os.Hostname() - if err != nil { - return nil, errors.Wrap(err, "unable to get hostname used to initialise default querier ID") - } - cfg.QuerierID = hostname - } - - resolver, err := naming.NewDNSResolverWithFreq(cfg.DNSLookupDuration) - if err != nil { - return nil, err - } - - watcher, err := resolver.Resolve(cfg.Address) - if err != nil { - return nil, err - } - - w := &worker{ - cfg: cfg, - querierCfg: querierCfg, - log: log, - server: server, - watcher: watcher, - managers: map[string]*frontendManager{}, - } - return services.NewBasicService(nil, w.watchDNSLoop, w.stopping), nil -} - -func (w *worker) stopping(_ error) error { - // wait until all per-address workers are done. This is only called after watchDNSLoop exits. - for _, mgr := range w.managers { - mgr.stop() - } - return nil -} - -// watchDNSLoop watches for changes in DNS and starts or stops workers. -func (w *worker) watchDNSLoop(servCtx context.Context) error { - go func() { - // Close the watcher, when this service is asked to stop. - // Closing the watcher makes watchDNSLoop exit, since it only iterates on watcher updates, and has no other - // way to stop. We cannot close the watcher in `stopping` method, because it is only called *after* - // watchDNSLoop exits. - <-servCtx.Done() - w.watcher.Close() - }() - - for { - updates, err := w.watcher.Next() - if err != nil { - // watcher.Next returns error when Close is called, but we call Close when our context is done. - // we don't want to report error in that case. - if servCtx.Err() != nil { - return nil - } - return errors.Wrapf(err, "error from DNS watcher") - } - - for _, update := range updates { - switch update.Op { - case naming.Add: - level.Debug(w.log).Log("msg", "adding connection", "addr", update.Addr) - conn, err := w.connect(servCtx, update.Addr) - if err != nil { - level.Error(w.log).Log("msg", "error connecting", "addr", update.Addr, "err", err) - continue - } - - w.managers[update.Addr] = newFrontendManager(servCtx, w.log, w.server, conn, NewFrontendClient(conn), w.cfg.GRPCClientConfig, w.cfg.QuerierID) - - case naming.Delete: - level.Debug(w.log).Log("msg", "removing connection", "addr", update.Addr) - if mgr, ok := w.managers[update.Addr]; ok { - mgr.stop() - delete(w.managers, update.Addr) - } - - default: - return fmt.Errorf("unknown op: %v", update.Op) - } - } - - w.resetConcurrency() - } -} - -func (w *worker) connect(ctx context.Context, address string) (*grpc.ClientConn, error) { - opts, err := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) - if err != nil { - return nil, err - } - - conn, err := grpc.DialContext(ctx, address, opts...) - if err != nil { - return nil, err - } - return conn, nil -} - -func (w *worker) resetConcurrency() { - addresses := make([]string, 0, len(w.managers)) - for addr := range w.managers { - addresses = append(addresses, addr) - } - rand.Shuffle(len(addresses), func(i, j int) { addresses[i], addresses[j] = addresses[j], addresses[i] }) - - totalConcurrency := 0 - for i, addr := range addresses { - concurrentRequests := w.concurrency(i, addr) - totalConcurrency += concurrentRequests - - if mgr, ok := w.managers[addr]; ok { - mgr.concurrentRequests(concurrentRequests) - } else { - level.Error(w.log).Log("msg", "address not found in managers map. this should not happen", "addr", addr) - } - } - - if totalConcurrency > w.querierCfg.MaxConcurrent { - level.Warn(w.log).Log("msg", "total worker concurrency is greater than promql max concurrency. queries may be queued in the querier which reduces QOS") - } -} - -func (w *worker) concurrency(index int, addr string) int { - concurrentRequests := 0 - - if w.cfg.MatchMaxConcurrency { - concurrentRequests = w.querierCfg.MaxConcurrent / len(w.managers) - - // If max concurrency does not evenly divide into our frontends a subset will be chosen - // to receive an extra connection. Frontend addresses were shuffled above so this will be a - // random selection of frontends. - if index < w.querierCfg.MaxConcurrent%len(w.managers) { - level.Warn(w.log).Log("msg", "max concurrency is not evenly divisible across query frontends. adding an extra connection", "addr", addr) - concurrentRequests++ - } - } else { - concurrentRequests = w.cfg.Parallelism - } - - // If concurrentRequests is 0 then w.querierCfg.MaxConcurrent is less than the total number of - // query frontends. In order to prevent accidentally starving a frontend we are just going to - // always connect once to every frontend. This is dangerous b/c we may start exceeding promql - // max concurrency. - if concurrentRequests == 0 { - concurrentRequests = 1 - } - - return concurrentRequests -} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker_frontend_manager.go b/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker_frontend_manager.go deleted file mode 100644 index 194abe108038..000000000000 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/worker_frontend_manager.go +++ /dev/null @@ -1,172 +0,0 @@ -package frontend - -import ( - "context" - "fmt" - "io" - "net/http" - "sync" - "time" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/httpgrpc/server" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/grpcclient" -) - -var ( - backoffConfig = util.BackoffConfig{ - MinBackoff: 50 * time.Millisecond, - MaxBackoff: 1 * time.Second, - } -) - -type frontendManager struct { - server *server.Server - connection io.Closer - client FrontendClient - clientCfg grpcclient.ConfigWithTLS - querierID string - - log log.Logger - - workerCancels []context.CancelFunc - serverCtx context.Context - wg sync.WaitGroup - currentProcessors *atomic.Int32 -} - -func newFrontendManager(serverCtx context.Context, log log.Logger, server *server.Server, connection io.Closer, client FrontendClient, clientCfg grpcclient.ConfigWithTLS, querierID string) *frontendManager { - f := &frontendManager{ - log: log, - connection: connection, - client: client, - clientCfg: clientCfg, - server: server, - serverCtx: serverCtx, - currentProcessors: atomic.NewInt32(0), - querierID: querierID, - } - - return f -} - -func (f *frontendManager) stop() { - f.concurrentRequests(0) - f.wg.Wait() - _ = f.connection.Close() -} - -func (f *frontendManager) concurrentRequests(n int) { - if n < 0 { - n = 0 - } - - for len(f.workerCancels) < n { - ctx, cancel := context.WithCancel(f.serverCtx) - f.workerCancels = append(f.workerCancels, cancel) - - go f.runOne(ctx) - } - - for len(f.workerCancels) > n { - var cancel context.CancelFunc - cancel, f.workerCancels = f.workerCancels[0], f.workerCancels[1:] - cancel() - } -} - -// runOne loops, trying to establish a stream to the frontend to begin -// request processing. -func (f *frontendManager) runOne(ctx context.Context) { - f.wg.Add(1) - defer f.wg.Done() - - f.currentProcessors.Inc() - defer f.currentProcessors.Dec() - - backoff := util.NewBackoff(ctx, backoffConfig) - for backoff.Ongoing() { - c, err := f.client.Process(ctx) - if err != nil { - level.Error(f.log).Log("msg", "error contacting frontend", "err", err) - backoff.Wait() - continue - } - - if err := f.process(c); err != nil { - level.Error(f.log).Log("msg", "error processing requests", "err", err) - backoff.Wait() - continue - } - - backoff.Reset() - } -} - -// process loops processing requests on an established stream. -func (f *frontendManager) process(c Frontend_ProcessClient) error { - // Build a child context so we can cancel a query when the stream is closed. - ctx, cancel := context.WithCancel(c.Context()) - defer cancel() - - for { - request, err := c.Recv() - if err != nil { - return err - } - - switch request.Type { - case HTTP_REQUEST: - // Handle the request on a "background" goroutine, so we go back to - // blocking on c.Recv(). This allows us to detect the stream closing - // and cancel the query. We don't actually handle queries in parallel - // here, as we're running in lock step with the server - each Recv is - // paired with a Send. - go f.runRequest(ctx, request.HttpRequest, func(response *httpgrpc.HTTPResponse) error { - return c.Send(&ClientToFrontend{HttpResponse: response}) - }) - - case GET_ID: - err := c.Send(&ClientToFrontend{ClientID: f.querierID}) - if err != nil { - return err - } - - default: - return fmt.Errorf("unknown request type: %v", request.Type) - } - } -} - -func (f *frontendManager) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, sendHTTPResponse func(response *httpgrpc.HTTPResponse) error) { - response, err := f.server.Handle(ctx, request) - if err != nil { - var ok bool - response, ok = httpgrpc.HTTPResponseFromError(err) - if !ok { - response = &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(err.Error()), - } - } - } - - // Ensure responses that are too big are not retried. - if len(response.Body) >= f.clientCfg.GRPC.MaxSendMsgSize { - errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), f.clientCfg.GRPC.MaxSendMsgSize) - response = &httpgrpc.HTTPResponse{ - Code: http.StatusRequestEntityTooLarge, - Body: []byte(errMsg), - } - level.Error(f.log).Log("msg", "error processing query", "err", errMsg) - } - - if err := sendHTTPResponse(response); err != nil { - level.Error(f.log).Log("msg", "error processing requests", "err", err) - } -} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go b/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go index 91a108c89e83..7fa7042787ef 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/querier.go @@ -39,6 +39,7 @@ type Config struct { IngesterStreaming bool `yaml:"ingester_streaming"` MaxSamples int `yaml:"max_samples"` QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"` + QueryStoreForLabels bool `yaml:"query_store_for_labels_enabled"` // QueryStoreAfter the time after which queries should also be sent to the store and not just ingesters. QueryStoreAfter time.Duration `yaml:"query_store_after"` @@ -71,6 +72,7 @@ type Config struct { var ( errBadLookbackConfigs = errors.New("bad settings, query_store_after >= query_ingesters_within which can result in queries not being sent") errShuffleShardingLookbackLessThanQueryStoreAfter = errors.New("the shuffle-sharding lookback period should be greater or equal than the configured 'query store after'") + errEmptyTimeRange = errors.New("empty time range") ) // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -83,6 +85,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.IngesterStreaming, "querier.ingester-streaming", true, "Use streaming RPCs to query ingester.") f.IntVar(&cfg.MaxSamples, "querier.max-samples", 50e6, "Maximum number of samples a single query can load into memory.") f.DurationVar(&cfg.QueryIngestersWithin, "querier.query-ingesters-within", 0, "Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester.") + f.BoolVar(&cfg.QueryStoreForLabels, "querier.query-store-for-labels-enabled", false, "Query long-term store for series, label values and label names APIs. Works only with blocks engine.") f.DurationVar(&cfg.MaxQueryIntoFuture, "querier.max-query-into-future", 10*time.Minute, "Maximum duration into the future you can query. 0 to disable.") f.DurationVar(&cfg.DefaultEvaluationInterval, "querier.default-evaluation-interval", time.Minute, "The default evaluation interval or step size for subqueries.") f.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 0, "The time after which a metric should only be queried from storage and not just ingesters. 0 means all queries are sent to store. When running the blocks storage, if this option is enabled, the time range of the query sent to the store will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") @@ -204,24 +207,27 @@ func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { now := time.Now() - if cfg.MaxQueryIntoFuture > 0 { - maxQueryTime := util.TimeToMillis(now.Add(cfg.MaxQueryIntoFuture)) + userID, err := user.ExtractOrgID(ctx) + if err != nil { + return nil, err + } - if mint > maxQueryTime { - return storage.NoopQuerier(), nil - } - if maxt > maxQueryTime { - maxt = maxQueryTime - } + mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, limits, cfg.MaxQueryIntoFuture) + if err == errEmptyTimeRange { + return storage.NoopQuerier(), nil + } else if err != nil { + return nil, err } q := querier{ - ctx: ctx, - mint: mint, - maxt: maxt, - chunkIterFn: chunkIterFn, - tombstonesLoader: tombstonesLoader, - limits: limits, + ctx: ctx, + mint: mint, + maxt: maxt, + chunkIterFn: chunkIterFn, + tombstonesLoader: tombstonesLoader, + limits: limits, + maxQueryIntoFuture: cfg.MaxQueryIntoFuture, + queryStoreForLabels: cfg.QueryStoreForLabels, } dqr, err := distributor.Querier(ctx, mint, maxt) @@ -263,8 +269,10 @@ type querier struct { ctx context.Context mint, maxt int64 - tombstonesLoader *purger.TombstonesLoader - limits *validation.Overrides + tombstonesLoader *purger.TombstonesLoader + limits *validation.Overrides + maxQueryIntoFuture time.Duration + queryStoreForLabels bool } // Select implements storage.Querier interface. @@ -283,7 +291,9 @@ func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Mat // querying the long-term storage. // Also, in the recent versions of Prometheus, we pass in the hint but with Func set to "series". // See: https://github.com/prometheus/prometheus/pull/8050 - if sp == nil || sp.Func == "series" { + if (sp == nil || sp.Func == "series") && !q.queryStoreForLabels { + // In this case, the query time range has already been validated when the querier has been + // created. return q.metadataQuerier.Select(true, sp, matchers...) } @@ -292,13 +302,30 @@ func (q querier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Mat return storage.ErrSeriesSet(err) } - // Validate query time range. - startTime := model.Time(sp.Start) - endTime := model.Time(sp.End) + // Validate query time range. Even if the time range has already been validated when we created + // the querier, we need to check it again here because the time range specified in hints may be + // different. + startMs, endMs, err := validateQueryTimeRange(ctx, userID, sp.Start, sp.End, q.limits, q.maxQueryIntoFuture) + if err == errEmptyTimeRange { + return storage.NoopSeriesSet() + } else if err != nil { + return storage.ErrSeriesSet(err) + } + + // The time range may have been manipulated during the validation, + // so we make sure changes are reflected back to hints. + sp.Start = startMs + sp.End = endMs + + startTime := model.Time(startMs) + endTime := model.Time(endMs) + + // Validate query time range. This validation should be done only for instant / range queries and + // NOT for metadata queries (series, labels) because the query-frontend doesn't support splitting + // of such queries. if maxQueryLength := q.limits.MaxQueryLength(userID); maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { limitErr := validation.LimitError(fmt.Sprintf(validation.ErrQueryTooLong, endTime.Sub(startTime), maxQueryLength)) return storage.ErrSeriesSet(limitErr) - } tombstones, err := q.tombstonesLoader.GetPendingTombstonesForInterval(userID, startTime, endTime) @@ -475,3 +502,43 @@ func UseBeforeTimestampQueryable(queryable storage.Queryable, ts time.Time) Quer ts: t, } } + +func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs int64, limits *validation.Overrides, maxQueryIntoFuture time.Duration) (int64, int64, error) { + now := model.Now() + startTime := model.Time(startMs) + endTime := model.Time(endMs) + + // Clamp time range based on max query into future. + if maxQueryIntoFuture > 0 && endTime.After(now.Add(maxQueryIntoFuture)) { + origEndTime := endTime + endTime = now.Add(maxQueryIntoFuture) + + // Make sure to log it in traces to ease debugging. + level.Debug(spanlogger.FromContext(ctx)).Log( + "msg", "the end time of the query has been manipulated because of the 'max query into future' setting", + "original", util.FormatTimeModel(origEndTime), + "updated", util.FormatTimeModel(endTime)) + + if endTime.Before(startTime) { + return 0, 0, errEmptyTimeRange + } + } + + // Clamp the time range based on the max query lookback. + if maxQueryLookback := limits.MaxQueryLookback(userID); maxQueryLookback > 0 && startTime.Before(now.Add(-maxQueryLookback)) { + origStartTime := startTime + startTime = now.Add(-maxQueryLookback) + + // Make sure to log it in traces to ease debugging. + level.Debug(spanlogger.FromContext(ctx)).Log( + "msg", "the start time of the query has been manipulated because of the 'max query lookback' setting", + "original", util.FormatTimeModel(origStartTime), + "updated", util.FormatTimeModel(startTime)) + + if endTime.Before(startTime) { + return 0, 0, errEmptyTimeRange + } + } + + return int64(startTime), int64(endTime), nil +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/limits.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/limits.go index 54bb46f6d9c7..7dc37b30f7dd 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/limits.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/limits.go @@ -5,107 +5,92 @@ import ( "net/http" "time" + "github.com/go-kit/kit/log/level" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/spanlogger" "github.com/cortexproject/cortex/pkg/util/validation" ) // Limits allows us to specify per-tenant runtime limits on the behavior of // the query handling code. type Limits interface { + // MaxQueryLookback returns the max lookback period of queries. + MaxQueryLookback(userID string) time.Duration + + // MaxQueryLength returns the limit of the length (in time) of a query. MaxQueryLength(string) time.Duration + + // MaxQueryParallelism returns the limit to the number of split queries the + // frontend will process in parallel. MaxQueryParallelism(string) int + + // MaxCacheFreshness returns the period after which results are cacheable, + // to prevent caching of very recent results. MaxCacheFreshness(string) time.Duration } -type limits struct { +type limitsMiddleware struct { Limits next Handler } -// LimitsMiddleware creates a new Middleware that invalidates large queries based on Limits interface. -func LimitsMiddleware(l Limits) Middleware { +// NewLimitsMiddleware creates a new Middleware that enforces query limits. +func NewLimitsMiddleware(l Limits) Middleware { return MiddlewareFunc(func(next Handler) Handler { - return limits{ + return limitsMiddleware{ next: next, Limits: l, } }) } -func (l limits) Do(ctx context.Context, r Request) (Response, error) { - userid, err := user.ExtractOrgID(ctx) +func (l limitsMiddleware) Do(ctx context.Context, r Request) (Response, error) { + log, ctx := spanlogger.New(ctx, "limits") + defer log.Finish() + + userID, err := user.ExtractOrgID(ctx) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } - maxQueryLen := l.MaxQueryLength(userid) - queryLen := timestamp.Time(r.GetEnd()).Sub(timestamp.Time(r.GetStart())) - if maxQueryLen > 0 && queryLen > maxQueryLen { - return nil, httpgrpc.Errorf(http.StatusBadRequest, validation.ErrQueryTooLong, queryLen, maxQueryLen) - } - return l.next.Do(ctx, r) -} + // Clamp the time range based on the max query lookback. + if maxQueryLookback := l.MaxQueryLookback(userID); maxQueryLookback > 0 { + minStartTime := util.TimeToMillis(time.Now().Add(-maxQueryLookback)) -// RequestResponse contains a request response and the respective request that was used. -type RequestResponse struct { - Request Request - Response Response -} + if r.GetEnd() < minStartTime { + // The request is fully outside the allowed range, so we can return an + // empty response. + level.Debug(log).Log( + "msg", "skipping the execution of the query because its time range is before the 'max query lookback' setting", + "reqStart", util.FormatTimeMillis(r.GetStart()), + "redEnd", util.FormatTimeMillis(r.GetEnd()), + "maxQueryLookback", maxQueryLookback) -// DoRequests executes a list of requests in parallel. The limits parameters is used to limit parallelism per single request. -func DoRequests(ctx context.Context, downstream Handler, reqs []Request, limits Limits) ([]RequestResponse, error) { - userid, err := user.ExtractOrgID(ctx) - if err != nil { - return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) - } + return NewEmptyPrometheusResponse(), nil + } - // If one of the requests fail, we want to be able to cancel the rest of them. - ctx, cancel := context.WithCancel(ctx) - defer cancel() + if r.GetStart() < minStartTime { + // Replace the start time in the request. + level.Debug(log).Log( + "msg", "the start time of the query has been manipulated because of the 'max query lookback' setting", + "original", util.FormatTimeMillis(r.GetStart()), + "updated", util.FormatTimeMillis(minStartTime)) - // Feed all requests to a bounded intermediate channel to limit parallelism. - intermediate := make(chan Request) - go func() { - for _, req := range reqs { - intermediate <- req + r = r.WithStartEnd(minStartTime, r.GetEnd()) } - close(intermediate) - }() - - respChan, errChan := make(chan RequestResponse), make(chan error) - parallelism := limits.MaxQueryParallelism(userid) - if parallelism > len(reqs) { - parallelism = len(reqs) - } - for i := 0; i < parallelism; i++ { - go func() { - for req := range intermediate { - resp, err := downstream.Do(ctx, req) - if err != nil { - errChan <- err - } else { - respChan <- RequestResponse{req, resp} - } - } - }() } - resps := make([]RequestResponse, 0, len(reqs)) - var firstErr error - for range reqs { - select { - case resp := <-respChan: - resps = append(resps, resp) - case err := <-errChan: - if firstErr == nil { - cancel() - firstErr = err - } + // Enforce the max query length. + if maxQueryLength := l.MaxQueryLength(userID); maxQueryLength > 0 { + queryLen := timestamp.Time(r.GetEnd()).Sub(timestamp.Time(r.GetStart())) + if queryLen > maxQueryLength { + return nil, httpgrpc.Errorf(http.StatusBadRequest, validation.ErrQueryTooLong, queryLen, maxQueryLength) } } - return resps, firstErr + return l.next.Do(ctx, r) } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/query_range.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/query_range.go index 1071dd95f868..6e146a98e5f1 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/query_range.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/query_range.go @@ -76,7 +76,7 @@ type Request interface { // GetCachingOptions returns the caching options. GetCachingOptions() CachingOptions // WithStartEnd clone the current request with different start and end timestamp. - WithStartEnd(int64, int64) Request + WithStartEnd(startTime int64, endTime int64) Request // WithQuery clone the current request with a different query. WithQuery(string) Request proto.Message @@ -135,15 +135,20 @@ func (resp *PrometheusResponse) minTime() int64 { return result[0].Samples[0].TimestampMs } +// NewEmptyPrometheusResponse returns an empty successful Prometheus query range response. +func NewEmptyPrometheusResponse() *PrometheusResponse { + return &PrometheusResponse{ + Status: StatusSuccess, + Data: PrometheusData{ + ResultType: model.ValMatrix.String(), + Result: []SampleStream{}, + }, + } +} + func (prometheusCodec) MergeResponse(responses ...Response) (Response, error) { if len(responses) == 0 { - return &PrometheusResponse{ - Status: StatusSuccess, - Data: PrometheusData{ - ResultType: model.ValMatrix.String(), - Result: []SampleStream{}, - }, - }, nil + return NewEmptyPrometheusResponse(), nil } promResponses := make([]*PrometheusResponse, 0, len(responses)) diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/retry.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/retry.go index 38d3bb15c9d3..32d16b921713 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/retry.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/retry.go @@ -8,6 +8,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/httpgrpc" + + "github.com/cortexproject/cortex/pkg/util" ) type RetryMiddlewareMetrics struct { @@ -68,7 +70,7 @@ func (r retry) Do(ctx context.Context, req Request) (Response, error) { httpResp, ok := httpgrpc.HTTPResponseFromError(err) if !ok || httpResp.Code/100 == 5 { lastErr = err - level.Error(r.log).Log("msg", "error processing request", "try", tries, "err", err) + level.Error(util.WithContext(ctx, r.log)).Log("msg", "error processing request", "try", tries, "err", err) continue } diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/roundtrip.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/roundtrip.go index e890c218455d..b15e729f7a2d 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/roundtrip.go +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/roundtrip.go @@ -34,7 +34,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/querier/frontend" ) const day = 24 * time.Hour @@ -126,6 +125,17 @@ func MergeMiddlewares(middleware ...Middleware) Middleware { }) } +// Tripperware is a signature for all http client-side middleware. +type Tripperware func(http.RoundTripper) http.RoundTripper + +// RoundTripFunc is to http.RoundTripper what http.HandlerFunc is to http.Handler. +type RoundTripFunc func(*http.Request) (*http.Response, error) + +// RoundTrip implements http.RoundTripper. +func (f RoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + // NewTripperware returns a Tripperware configured with middlewares to limit, align, split, retry and cache requests. func NewTripperware( cfg Config, @@ -138,7 +148,7 @@ func NewTripperware( minShardingLookback time.Duration, registerer prometheus.Registerer, cacheGenNumberLoader CacheGenNumberLoader, -) (frontend.Tripperware, cache.Cache, error) { +) (Tripperware, cache.Cache, error) { // Per tenant query metrics. queriesPerTenant := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: "cortex", @@ -149,7 +159,7 @@ func NewTripperware( // Metric used to keep track of each middleware execution duration. metrics := NewInstrumentMiddlewareMetrics(registerer) - queryRangeMiddleware := []Middleware{LimitsMiddleware(limits)} + queryRangeMiddleware := []Middleware{NewLimitsMiddleware(limits)} if cfg.AlignQueriesWithStep { queryRangeMiddleware = append(queryRangeMiddleware, InstrumentMiddleware("step_align", metrics), StepAlignMiddleware) } @@ -196,11 +206,11 @@ func NewTripperware( queryRangeMiddleware = append(queryRangeMiddleware, InstrumentMiddleware("retry", metrics), NewRetryMiddleware(log, cfg.MaxRetries, NewRetryMiddlewareMetrics(registerer))) } - return frontend.Tripperware(func(next http.RoundTripper) http.RoundTripper { + return func(next http.RoundTripper) http.RoundTripper { // Finally, if the user selected any query range middleware, stitch it in. if len(queryRangeMiddleware) > 0 { queryrange := NewRoundTripper(next, codec, queryRangeMiddleware...) - return frontend.RoundTripFunc(func(r *http.Request) (*http.Response, error) { + return RoundTripFunc(func(r *http.Request) (*http.Response, error) { isQueryRange := strings.HasSuffix(r.URL.Path, "/query_range") op := "query" if isQueryRange { @@ -221,7 +231,7 @@ func NewTripperware( }) } return next - }), c, nil + }, c, nil } type roundTripper struct { diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/util.go b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/util.go new file mode 100644 index 000000000000..6324131b5334 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/queryrange/util.go @@ -0,0 +1,70 @@ +package queryrange + +import ( + "context" + "net/http" + + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/user" +) + +// RequestResponse contains a request response and the respective request that was used. +type RequestResponse struct { + Request Request + Response Response +} + +// DoRequests executes a list of requests in parallel. The limits parameters is used to limit parallelism per single request. +func DoRequests(ctx context.Context, downstream Handler, reqs []Request, limits Limits) ([]RequestResponse, error) { + userid, err := user.ExtractOrgID(ctx) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + + // If one of the requests fail, we want to be able to cancel the rest of them. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Feed all requests to a bounded intermediate channel to limit parallelism. + intermediate := make(chan Request) + go func() { + for _, req := range reqs { + intermediate <- req + } + close(intermediate) + }() + + respChan, errChan := make(chan RequestResponse), make(chan error) + parallelism := limits.MaxQueryParallelism(userid) + if parallelism > len(reqs) { + parallelism = len(reqs) + } + for i := 0; i < parallelism; i++ { + go func() { + for req := range intermediate { + resp, err := downstream.Do(ctx, req) + if err != nil { + errChan <- err + } else { + respChan <- RequestResponse{req, resp} + } + } + }() + } + + resps := make([]RequestResponse, 0, len(reqs)) + var firstErr error + for range reqs { + select { + case resp := <-respChan: + resps = append(resps, resp) + case err := <-errChan: + if firstErr == nil { + cancel() + firstErr = err + } + } + } + + return resps, firstErr +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/worker/frontend_processor.go b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/frontend_processor.go new file mode 100644 index 000000000000..318f66cb2183 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/frontend_processor.go @@ -0,0 +1,127 @@ +package worker + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/weaveworks/common/httpgrpc" + "google.golang.org/grpc" + + "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" + "github.com/cortexproject/cortex/pkg/util" +) + +var ( + processorBackoffConfig = util.BackoffConfig{ + MinBackoff: 50 * time.Millisecond, + MaxBackoff: 1 * time.Second, + } +) + +func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger) processor { + return &frontendProcessor{ + log: log, + handler: handler, + maxMessageSize: cfg.GRPCClientConfig.GRPC.MaxSendMsgSize, + querierID: cfg.QuerierID, + } +} + +// Handles incoming queries from frontend. +type frontendProcessor struct { + handler RequestHandler + maxMessageSize int + querierID string + + log log.Logger +} + +// runOne loops, trying to establish a stream to the frontend to begin request processing. +func (fp *frontendProcessor) processQueriesOnSingleStream(ctx context.Context, conn *grpc.ClientConn, address string) { + client := frontendv1pb.NewFrontendClient(conn) + + backoff := util.NewBackoff(ctx, processorBackoffConfig) + for backoff.Ongoing() { + c, err := client.Process(ctx) + if err != nil { + level.Error(fp.log).Log("msg", "error contacting frontend", "address", address, "err", err) + backoff.Wait() + continue + } + + if err := fp.process(c); err != nil { + level.Error(fp.log).Log("msg", "error processing requests", "address", address, "err", err) + backoff.Wait() + continue + } + + backoff.Reset() + } +} + +// process loops processing requests on an established stream. +func (fp *frontendProcessor) process(c frontendv1pb.Frontend_ProcessClient) error { + // Build a child context so we can cancel a query when the stream is closed. + ctx, cancel := context.WithCancel(c.Context()) + defer cancel() + + for { + request, err := c.Recv() + if err != nil { + return err + } + + switch request.Type { + case frontendv1pb.HTTP_REQUEST: + // Handle the request on a "background" goroutine, so we go back to + // blocking on c.Recv(). This allows us to detect the stream closing + // and cancel the query. We don't actually handle queries in parallel + // here, as we're running in lock step with the server - each Recv is + // paired with a Send. + go fp.runRequest(ctx, request.HttpRequest, func(response *httpgrpc.HTTPResponse) error { + return c.Send(&frontendv1pb.ClientToFrontend{HttpResponse: response}) + }) + + case frontendv1pb.GET_ID: + err := c.Send(&frontendv1pb.ClientToFrontend{ClientID: fp.querierID}) + if err != nil { + return err + } + + default: + return fmt.Errorf("unknown request type: %v", request.Type) + } + } +} + +func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, sendHTTPResponse func(response *httpgrpc.HTTPResponse) error) { + response, err := fp.handler.Handle(ctx, request) + if err != nil { + var ok bool + response, ok = httpgrpc.HTTPResponseFromError(err) + if !ok { + response = &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + } + } + + // Ensure responses that are too big are not retried. + if len(response.Body) >= fp.maxMessageSize { + errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), fp.maxMessageSize) + response = &httpgrpc.HTTPResponse{ + Code: http.StatusRequestEntityTooLarge, + Body: []byte(errMsg), + } + level.Error(fp.log).Log("msg", "error processing query", "err", errMsg) + } + + if err := sendHTTPResponse(response); err != nil { + level.Error(fp.log).Log("msg", "error processing requests", "err", err) + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/worker/processor_manager.go b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/processor_manager.go new file mode 100644 index 000000000000..8a68c310e205 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/processor_manager.go @@ -0,0 +1,75 @@ +package worker + +import ( + "context" + "sync" + + "go.uber.org/atomic" + "google.golang.org/grpc" +) + +// Manages processor goroutines for single grpc connection. +type processorManager struct { + p processor + conn *grpc.ClientConn + address string + + // Main context to control all goroutines. + ctx context.Context + wg sync.WaitGroup + + // Cancel functions for individual goroutines. + cancelsMu sync.Mutex + cancels []context.CancelFunc + + currentProcessors *atomic.Int32 +} + +func newProcessorManager(ctx context.Context, p processor, conn *grpc.ClientConn, address string) *processorManager { + return &processorManager{ + p: p, + ctx: ctx, + conn: conn, + address: address, + currentProcessors: atomic.NewInt32(0), + } +} + +func (pm *processorManager) stop() { + // Stop all goroutines. + pm.concurrency(0) + + // Wait until they finish. + pm.wg.Wait() + + _ = pm.conn.Close() +} + +func (pm *processorManager) concurrency(n int) { + pm.cancelsMu.Lock() + defer pm.cancelsMu.Unlock() + + if n < 0 { + n = 0 + } + + for len(pm.cancels) < n { + ctx, cancel := context.WithCancel(pm.ctx) + pm.cancels = append(pm.cancels, cancel) + + pm.wg.Add(1) + go func() { + defer pm.wg.Done() + + pm.currentProcessors.Inc() + defer pm.currentProcessors.Dec() + + pm.p.processQueriesOnSingleStream(ctx, pm.conn, pm.address) + }() + } + + for len(pm.cancels) > n { + pm.cancels[0]() + pm.cancels = pm.cancels[1:] + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/worker/scheduler_processor.go b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/scheduler_processor.go new file mode 100644 index 000000000000..55b0c6d84e45 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/scheduler_processor.go @@ -0,0 +1,210 @@ +package worker + +import ( + "context" + "fmt" + "net/http" + "time" + + otgrpc "github.com/opentracing-contrib/go-grpc" + "github.com/weaveworks/common/user" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/middleware" + "google.golang.org/grpc" + "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" + "github.com/cortexproject/cortex/pkg/ring/client" + "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/grpcutil" + cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware" + "github.com/cortexproject/cortex/pkg/util/services" +) + +func newSchedulerProcessor(cfg Config, handler RequestHandler, log log.Logger, reg prometheus.Registerer) (*schedulerProcessor, []services.Service) { + p := &schedulerProcessor{ + log: log, + handler: handler, + maxMessageSize: cfg.GRPCClientConfig.GRPC.MaxSendMsgSize, + querierID: cfg.QuerierID, + grpcConfig: cfg.GRPCClientConfig, + + frontendClientRequestDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_querier_query_frontend_request_duration_seconds", + Help: "Time spend doing requests to frontend.", + Buckets: prometheus.ExponentialBuckets(0.001, 4, 6), + }, []string{"operation", "status_code"}), + } + + frontendClientsGauge := promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "cortex_querier_query_frontend_clients", + Help: "The current number of clients connected to query-frontend.", + }) + + poolConfig := client.PoolConfig{ + CheckInterval: 5 * time.Second, + HealthCheckEnabled: true, + HealthCheckTimeout: 1 * time.Second, + } + + p.frontendPool = client.NewPool("frontend", poolConfig, nil, p.createFrontendClient, frontendClientsGauge, log) + return p, []services.Service{p.frontendPool} +} + +// Handles incoming queries from query-scheduler. +type schedulerProcessor struct { + log log.Logger + handler RequestHandler + grpcConfig grpcclient.ConfigWithTLS + maxMessageSize int + querierID string + + frontendPool *client.Pool + frontendClientRequestDuration *prometheus.HistogramVec +} + +func (sp *schedulerProcessor) processQueriesOnSingleStream(ctx context.Context, conn *grpc.ClientConn, address string) { + schedulerClient := schedulerpb.NewSchedulerForQuerierClient(conn) + + backoff := util.NewBackoff(ctx, processorBackoffConfig) + for backoff.Ongoing() { + c, err := schedulerClient.QuerierLoop(ctx) + if err == nil { + err = c.Send(&schedulerpb.QuerierToScheduler{QuerierID: sp.querierID}) + } + + if err != nil { + level.Error(sp.log).Log("msg", "error contacting scheduler", "err", err, "addr", address) + backoff.Wait() + continue + } + + if err := sp.querierLoop(c, address); err != nil { + level.Error(sp.log).Log("msg", "error processing requests from scheduler", "err", err, "addr", address) + backoff.Wait() + continue + } + + backoff.Reset() + } +} + +// process loops processing requests on an established stream. +func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_QuerierLoopClient, address string) error { + // Build a child context so we can cancel a query when the stream is closed. + ctx, cancel := context.WithCancel(c.Context()) + defer cancel() + + for { + request, err := c.Recv() + if err != nil { + return err + } + + // Handle the request on a "background" goroutine, so we go back to + // blocking on c.Recv(). This allows us to detect the stream closing + // and cancel the query. We don't actually handle queries in parallel + // here, as we're running in lock step with the server - each Recv is + // paired with a Send. + go func() { + // We need to inject user into context for sending response back. + ctx := user.InjectOrgID(ctx, request.UserID) + + tracer := opentracing.GlobalTracer() + // Ignore errors here. If we cannot get parent span, we just don't create new one. + parentSpanContext, _ := grpcutil.GetParentSpanForRequest(tracer, request.HttpRequest) + if parentSpanContext != nil { + queueSpan, spanCtx := opentracing.StartSpanFromContextWithTracer(ctx, tracer, "querier_processor_runRequest", opentracing.ChildOf(parentSpanContext)) + defer queueSpan.Finish() + + ctx = spanCtx + } + logger := util.WithContext(ctx, sp.log) + + sp.runRequest(ctx, logger, request.QueryID, request.FrontendAddress, request.HttpRequest) + + // Report back to scheduler that processing of the query has finished. + if err := c.Send(&schedulerpb.QuerierToScheduler{}); err != nil { + level.Error(logger).Log("msg", "error notifying scheduler about finished query", "err", err, "addr", address) + } + }() + } +} + +func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger, queryID uint64, frontendAddress string, request *httpgrpc.HTTPRequest) { + response, err := sp.handler.Handle(ctx, request) + if err != nil { + var ok bool + response, ok = httpgrpc.HTTPResponseFromError(err) + if !ok { + response = &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + } + } + + // Ensure responses that are too big are not retried. + if len(response.Body) >= sp.maxMessageSize { + level.Error(logger).Log("msg", "response larger than max message size", "size", len(response.Body), "maxMessageSize", sp.maxMessageSize) + + errMsg := fmt.Sprintf("response larger than the max message size (%d vs %d)", len(response.Body), sp.maxMessageSize) + response = &httpgrpc.HTTPResponse{ + Code: http.StatusRequestEntityTooLarge, + Body: []byte(errMsg), + } + } + + c, err := sp.frontendPool.GetClientFor(frontendAddress) + if err == nil { + // Response is empty and uninteresting. + _, err = c.(frontendv2pb.FrontendForQuerierClient).QueryResult(ctx, &frontendv2pb.QueryResultRequest{ + QueryID: queryID, + HttpResponse: response, + }) + } + if err != nil { + level.Error(logger).Log("msg", "error notifying frontend about finished query", "err", err, "frontend", frontendAddress) + } +} + +func (sp *schedulerProcessor) createFrontendClient(addr string) (client.PoolClient, error) { + opts, err := sp.grpcConfig.DialOption([]grpc.UnaryClientInterceptor{ + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), + middleware.ClientUserHeaderInterceptor, + cortex_middleware.PrometheusGRPCUnaryInstrumentation(sp.frontendClientRequestDuration), + }, nil) + + if err != nil { + return nil, err + } + + conn, err := grpc.Dial(addr, opts...) + if err != nil { + return nil, err + } + + return &frontendClient{ + FrontendForQuerierClient: frontendv2pb.NewFrontendForQuerierClient(conn), + HealthClient: grpc_health_v1.NewHealthClient(conn), + conn: conn, + }, nil +} + +type frontendClient struct { + frontendv2pb.FrontendForQuerierClient + grpc_health_v1.HealthClient + conn *grpc.ClientConn +} + +func (fc *frontendClient) Close() error { + return fc.conn.Close() +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/worker/worker.go b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/worker.go new file mode 100644 index 000000000000..f9d5bce9d982 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/querier/worker/worker.go @@ -0,0 +1,263 @@ +package worker + +import ( + "context" + "flag" + "os" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/weaveworks/common/httpgrpc" + "google.golang.org/grpc" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/services" +) + +type Config struct { + FrontendAddress string `yaml:"frontend_address"` + SchedulerAddress string `yaml:"scheduler_address"` + DNSLookupPeriod time.Duration `yaml:"dns_lookup_duration"` + + Parallelism int `yaml:"parallelism"` + MatchMaxConcurrency bool `yaml:"match_max_concurrent"` + MaxConcurrentRequests int `yaml:"-"` // Must be same as passed to PromQL Engine. + + QuerierID string `yaml:"id"` + + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` +} + +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&cfg.SchedulerAddress, "querier.scheduler-address", "", "Hostname (and port) of scheduler that querier will periodically resolve, connect to and receive queries from. If set, takes precedence over -querier.frontend-address.") + f.StringVar(&cfg.FrontendAddress, "querier.frontend-address", "", "Address of query frontend service, in host:port format. If -querier.scheduler-address is set as well, querier will use scheduler instead. If neither -querier.frontend-address or -querier.scheduler-address is set, queries must arrive via HTTP endpoint.") + + f.DurationVar(&cfg.DNSLookupPeriod, "querier.dns-lookup-period", 10*time.Second, "How often to query DNS for query-frontend or query-scheduler address.") + + f.IntVar(&cfg.Parallelism, "querier.worker-parallelism", 10, "Number of simultaneous queries to process per query-frontend or query-scheduler.") + f.BoolVar(&cfg.MatchMaxConcurrency, "querier.worker-match-max-concurrent", false, "Force worker concurrency to match the -querier.max-concurrent option. Overrides querier.worker-parallelism.") + f.StringVar(&cfg.QuerierID, "querier.id", "", "Querier ID, sent to frontend service to identify requests from the same querier. Defaults to hostname.") + + cfg.GRPCClientConfig.RegisterFlagsWithPrefix("querier.frontend-client", f) +} + +func (cfg *Config) Validate(log log.Logger) error { + return cfg.GRPCClientConfig.Validate(log) +} + +// Handler for HTTP requests wrapped in protobuf messages. +type RequestHandler interface { + Handle(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) +} + +// Single processor handles all streaming operations to query-frontend or query-scheduler to fetch queries +// and process them. +type processor interface { + // Each invocation of processQueriesOnSingleStream starts new streaming operation to query-frontend + // or query-scheduler to fetch queries and execute them. + // + // This method must react on context being finished, and stop when that happens. + // + // processorManager (not processor) is responsible for starting as many goroutines as needed for each connection. + processQueriesOnSingleStream(ctx context.Context, conn *grpc.ClientConn, address string) +} + +type querierWorker struct { + *services.BasicService + + cfg Config + log log.Logger + + processor processor + + subservices *services.Manager + + mu sync.Mutex + // Set to nil when stop is called... no more managers are created afterwards. + managers map[string]*processorManager +} + +func NewQuerierWorker(cfg Config, handler RequestHandler, log log.Logger, reg prometheus.Registerer) (services.Service, error) { + if cfg.QuerierID == "" { + hostname, err := os.Hostname() + if err != nil { + return nil, errors.Wrap(err, "failed to get hostname for configuring querier ID") + } + cfg.QuerierID = hostname + } + + var processor processor + var servs []services.Service + var address string + + switch { + case cfg.SchedulerAddress != "": + level.Info(log).Log("msg", "Starting querier worker connected to query-scheduler", "scheduler", cfg.SchedulerAddress) + + address = cfg.SchedulerAddress + processor, servs = newSchedulerProcessor(cfg, handler, log, reg) + + case cfg.FrontendAddress != "": + level.Info(log).Log("msg", "Starting querier worker connected to query-frontend", "frontend", cfg.FrontendAddress) + + address = cfg.FrontendAddress + processor = newFrontendProcessor(cfg, handler, log) + + default: + return nil, errors.New("no query-scheduler or query-frontend address") + } + + return newQuerierWorkerWithProcessor(cfg, log, processor, address, servs) +} + +func newQuerierWorkerWithProcessor(cfg Config, log log.Logger, processor processor, address string, servs []services.Service) (*querierWorker, error) { + f := &querierWorker{ + cfg: cfg, + log: log, + managers: map[string]*processorManager{}, + processor: processor, + } + + // Empty address is only used in tests, where individual targets are added manually. + if address != "" { + w, err := util.NewDNSWatcher(address, cfg.DNSLookupPeriod, f) + if err != nil { + return nil, err + } + + servs = append(servs, w) + } + + if len(servs) > 0 { + subservices, err := services.NewManager(servs...) + if err != nil { + return nil, errors.Wrap(err, "querier worker subservices") + } + + f.subservices = subservices + } + + f.BasicService = services.NewIdleService(f.starting, f.stopping) + return f, nil +} + +func (w *querierWorker) starting(ctx context.Context) error { + if w.subservices == nil { + return nil + } + return services.StartManagerAndAwaitHealthy(ctx, w.subservices) +} + +func (w *querierWorker) stopping(_ error) error { + // Stop all goroutines fetching queries. Note that in Stopping state, + // worker no longer creates new managers in AddressAdded method. + w.mu.Lock() + for _, m := range w.managers { + m.stop() + } + w.mu.Unlock() + + if w.subservices == nil { + return nil + } + + // Stop DNS watcher and services used by processor. + return services.StopManagerAndAwaitStopped(context.Background(), w.subservices) +} + +func (w *querierWorker) AddressAdded(address string) { + ctx := w.ServiceContext() + if ctx == nil || ctx.Err() != nil { + return + } + + w.mu.Lock() + defer w.mu.Unlock() + + if m := w.managers[address]; m != nil { + return + } + + level.Info(w.log).Log("msg", "adding connection", "addr", address) + conn, err := w.connect(context.Background(), address) + if err != nil { + level.Error(w.log).Log("msg", "error connecting", "addr", address, "err", err) + return + } + + w.managers[address] = newProcessorManager(ctx, w.processor, conn, address) + // Called with lock. + w.resetConcurrency() +} + +func (w *querierWorker) AddressRemoved(address string) { + level.Info(w.log).Log("msg", "removing connection", "addr", address) + + w.mu.Lock() + p := w.managers[address] + delete(w.managers, address) + w.mu.Unlock() + + if p != nil { + p.stop() + } +} + +// Must be called with lock. +func (w *querierWorker) resetConcurrency() { + totalConcurrency := 0 + index := 0 + + for _, m := range w.managers { + concurrency := 0 + + if w.cfg.MatchMaxConcurrency { + concurrency = w.cfg.MaxConcurrentRequests / len(w.managers) + + // If max concurrency does not evenly divide into our frontends a subset will be chosen + // to receive an extra connection. Frontend addresses were shuffled above so this will be a + // random selection of frontends. + if index < w.cfg.MaxConcurrentRequests%len(w.managers) { + level.Warn(w.log).Log("msg", "max concurrency is not evenly divisible across targets, adding an extra connection", "addr", m.address) + concurrency++ + } + } else { + concurrency = w.cfg.Parallelism + } + + // If concurrency is 0 then MaxConcurrentRequests is less than the total number of + // frontends/schedulers. In order to prevent accidentally starving a frontend or scheduler we are just going to + // always connect once to every target. This is dangerous b/c we may start exceeding PromQL + // max concurrency. + if concurrency == 0 { + concurrency = 1 + } + + totalConcurrency += concurrency + m.concurrency(concurrency) + index++ + } + + if totalConcurrency > w.cfg.MaxConcurrentRequests { + level.Warn(w.log).Log("msg", "total worker concurrency is greater than promql max concurrency. Queries may be queued in the querier which reduces QOS") + } +} + +func (w *querierWorker) connect(ctx context.Context, address string) (*grpc.ClientConn, error) { + // Because we only use single long-running method, it doesn't make sense to inject user ID, send over tracing or add metrics. + opts, err := w.cfg.GRPCClientConfig.DialOption(nil, nil) + if err != nil { + return nil, err + } + + conn, err := grpc.DialContext(ctx, address, opts...) + if err != nil { + return nil, err + } + return conn, nil +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/ring/client/ring_service_discovery.go b/vendor/github.com/cortexproject/cortex/pkg/ring/client/ring_service_discovery.go index 93958920fa8e..e0ab7ce64b93 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ring/client/ring_service_discovery.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ring/client/ring_service_discovery.go @@ -6,7 +6,7 @@ import ( func NewRingServiceDiscovery(r ring.ReadRing) PoolServiceDiscovery { return func() ([]string, error) { - replicationSet, err := r.GetAll(ring.Read) + replicationSet, err := r.GetAllHealthy(ring.Reporting) if err != nil { return nil, err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/ring/model.go b/vendor/github.com/cortexproject/cortex/pkg/ring/model.go index 19ab91c9e539..d9ebd78155c2 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ring/model.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ring/model.go @@ -165,6 +165,9 @@ func (i *IngesterDesc) IsHealthy(op Operation, heartbeatTimeout time.Duration) b case Ruler: healthy = i.State == ACTIVE + + case Compactor: + healthy = i.State == ACTIVE } return healthy && time.Since(time.Unix(i.Timestamp, 0)) <= heartbeatTimeout diff --git a/vendor/github.com/cortexproject/cortex/pkg/ring/ring.go b/vendor/github.com/cortexproject/cortex/pkg/ring/ring.go index 7192399d9909..6d2492c94efc 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ring/ring.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ring/ring.go @@ -44,7 +44,18 @@ type ReadRing interface { // buf is a slice to be overwritten for the return value // to avoid memory allocation; can be nil. Get(key uint32, op Operation, buf []IngesterDesc) (ReplicationSet, error) - GetAll(op Operation) (ReplicationSet, error) + + // GetAllHealthy returns all healthy instances in the ring, for the given operation. + // This function doesn't check if the quorum is honored, so doesn't fail if the number + // of unhealthy ingesters is greater than the tolerated max unavailable. + GetAllHealthy(op Operation) (ReplicationSet, error) + + // GetReplicationSetForOperation returns all instances where the input operation should be executed. + // The resulting ReplicationSet doesn't necessarily contains all healthy instances + // in the ring, but could contain the minimum set of instances required to execute + // the input operation. + GetReplicationSetForOperation(op Operation) (ReplicationSet, error) + ReplicationFactor() int IngesterCount() int @@ -75,7 +86,11 @@ const ( // BlocksRead is the operation run by the querier to query blocks via the store-gateway. BlocksRead - Ruler // Used for distributing rule groups between rulers. + // Ruler is the operation used for distributing rule groups between rulers. + Ruler + + // Compactor is the operation used for distributing tenants/blocks across compactors. + Compactor ) var ( @@ -85,6 +100,10 @@ var ( // ErrInstanceNotFound is the error returned when trying to get information for an instance // not registered within the ring. ErrInstanceNotFound = errors.New("instance not found in the ring") + + // ErrTooManyFailedIngesters is the error returned when there are too many failed ingesters for a + // specific operation. + ErrTooManyFailedIngesters = errors.New("too many failed ingesters") ) // Config for a Ring @@ -313,8 +332,30 @@ func (r *Ring) Get(key uint32, op Operation, buf []IngesterDesc) (ReplicationSet }, nil } -// GetAll returns all available ingesters in the ring. -func (r *Ring) GetAll(op Operation) (ReplicationSet, error) { +// GetAllHealthy implements ReadRing. +func (r *Ring) GetAllHealthy(op Operation) (ReplicationSet, error) { + r.mtx.RLock() + defer r.mtx.RUnlock() + + if r.ringDesc == nil || len(r.ringDesc.Ingesters) == 0 { + return ReplicationSet{}, ErrEmptyRing + } + + ingesters := make([]IngesterDesc, 0, len(r.ringDesc.Ingesters)) + for _, ingester := range r.ringDesc.Ingesters { + if r.IsHealthy(&ingester, op) { + ingesters = append(ingesters, ingester) + } + } + + return ReplicationSet{ + Ingesters: ingesters, + MaxErrors: 0, + }, nil +} + +// GetReplicationSetForOperation implements ReadRing. +func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, error) { r.mtx.RLock() defer r.mtx.RUnlock() @@ -339,7 +380,7 @@ func (r *Ring) GetAll(op Operation) (ReplicationSet, error) { } if len(ingesters) < numRequired { - return ReplicationSet{}, fmt.Errorf("too many failed ingesters") + return ReplicationSet{}, ErrTooManyFailedIngesters } return ReplicationSet{ diff --git a/vendor/github.com/cortexproject/cortex/pkg/ruler/api.go b/vendor/github.com/cortexproject/cortex/pkg/ruler/api.go index ffa6e9283659..cdf7b903dd3c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ruler/api.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ruler/api.go @@ -134,7 +134,7 @@ func NewAPI(r *Ruler, s rules.RuleStore) *API { func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { logger := util.WithContext(req.Context(), util.Logger) - userID, ctx, err := user.ExtractOrgIDFromHTTPRequest(req) + userID, err := user.ExtractOrgID(req.Context()) if err != nil || userID == "" { level.Error(logger).Log("msg", "error extracting org id from context", "err", err) respondError(logger, w, "no valid org id found") @@ -142,7 +142,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { } w.Header().Set("Content-Type", "application/json") - rgs, err := a.ruler.GetRules(ctx) + rgs, err := a.ruler.GetRules(req.Context()) if err != nil { respondError(logger, w, err.Error()) @@ -226,7 +226,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) { logger := util.WithContext(req.Context(), util.Logger) - userID, ctx, err := user.ExtractOrgIDFromHTTPRequest(req) + userID, err := user.ExtractOrgID(req.Context()) if err != nil || userID == "" { level.Error(logger).Log("msg", "error extracting org id from context", "err", err) respondError(logger, w, "no valid org id found") @@ -234,7 +234,7 @@ func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) { } w.Header().Set("Content-Type", "application/json") - rgs, err := a.ruler.GetRules(ctx) + rgs, err := a.ruler.GetRules(req.Context()) if err != nil { respondError(logger, w, err.Error()) diff --git a/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go b/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go index 8768832936ed..438f89331454 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go +++ b/vendor/github.com/cortexproject/cortex/pkg/ruler/ruler.go @@ -406,7 +406,7 @@ func (r *Ruler) run(ctx context.Context) error { var ringLastState ring.ReplicationSet if r.cfg.EnableSharding { - ringLastState, _ = r.ring.GetAll(ring.Ruler) + ringLastState, _ = r.ring.GetAllHealthy(ring.Ruler) ringTicker := time.NewTicker(util.DurationWithJitter(r.cfg.RingCheckPeriod, 0.2)) defer ringTicker.Stop() ringTickerChan = ringTicker.C @@ -422,7 +422,7 @@ func (r *Ruler) run(ctx context.Context) error { case <-ringTickerChan: // We ignore the error because in case of error it will return an empty // replication set which we use to compare with the previous state. - currRingState, _ := r.ring.GetAll(ring.Ruler) + currRingState, _ := r.ring.GetAllHealthy(ring.Ruler) if ring.HasReplicationSetChanged(ringLastState, currRingState) { ringLastState = currRingState @@ -684,7 +684,7 @@ func (r *Ruler) getLocalRules(userID string) ([]*GroupStateDesc, error) { } func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) { - rulers, err := r.ring.GetAll(ring.Ruler) + rulers, err := r.ring.GetReplicationSetForOperation(ring.Ruler) if err != nil { return nil, err } diff --git a/vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/queue.go b/vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/queue.go new file mode 100644 index 000000000000..342e2a26d8a7 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/queue.go @@ -0,0 +1,189 @@ +package queue + +import ( + "context" + "sync" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" +) + +var ( + ErrTooManyRequests = errors.New("too many outstanding requests") + ErrStopped = errors.New("queue is stopped") +) + +// UserIndex is opaque type that allows to resume iteration over users between successive calls +// of RequestQueue.GetNextRequestForQuerier method. +type UserIndex struct { + last int +} + +// Modify index to start iteration on the same user, for which last queue was returned. +func (ui UserIndex) ReuseLastUser() UserIndex { + if ui.last >= 0 { + return UserIndex{last: ui.last - 1} + } + return ui +} + +// FirstUser returns UserIndex that starts iteration over user queues from the very first user. +func FirstUser() UserIndex { + return UserIndex{last: -1} +} + +// Request stored into the queue. +type Request interface{} + +// RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers, +// and when querier asks for next request to handle (using GetNextRequestForQuerier), it returns requests +// in a fair fashion. +type RequestQueue struct { + connectedQuerierWorkers *atomic.Int32 + + mtx sync.Mutex + cond *sync.Cond // Notified when request is enqueued or dequeued, or querier is disconnected. + queues *queues + stopped bool + + queueLength *prometheus.GaugeVec // Per user. +} + +func NewRequestQueue(maxOutstandingPerTenant int, queueLength *prometheus.GaugeVec) *RequestQueue { + q := &RequestQueue{ + queues: newUserQueues(maxOutstandingPerTenant), + connectedQuerierWorkers: atomic.NewInt32(0), + queueLength: queueLength, + } + + q.cond = sync.NewCond(&q.mtx) + + return q +} + +// Puts the request into the queue. MaxQueries is user-specific value that specifies how many queriers can +// this user use (zero or negative = all queriers). It is passed to each EnqueueRequest, because it can change +// between calls. +// +// If request is successfully enqueued, successFn is called with the lock held, before any querier can receive the request. +func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers int, successFn func()) error { + q.mtx.Lock() + defer q.mtx.Unlock() + + if q.stopped { + return ErrStopped + } + + queue := q.queues.getOrAddQueue(userID, maxQueriers) + if queue == nil { + // This can only happen if userID is "". + return errors.New("no queue found") + } + + select { + case queue <- req: + q.queueLength.WithLabelValues(userID).Inc() + q.cond.Broadcast() + // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. + if successFn != nil { + successFn() + } + return nil + default: + return ErrTooManyRequests + } +} + +// GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests. +// By passing user index from previous call of this method, querier guarantees that it iterates over all users fairly. +// If querier finds that request from the user is already expired, it can get a request for the same user by using UserIndex.ReuseLastUser. +func (q *RequestQueue) GetNextRequestForQuerier(ctx context.Context, last UserIndex, querierID string) (Request, UserIndex, error) { + q.mtx.Lock() + defer q.mtx.Unlock() + + querierWait := false + +FindQueue: + // We need to wait if there are no users, or no pending requests for given querier. + for (q.queues.len() == 0 || querierWait) && ctx.Err() == nil && !q.stopped { + querierWait = false + q.cond.Wait() + } + + if q.stopped { + return nil, last, ErrStopped + } + + if err := ctx.Err(); err != nil { + return nil, last, err + } + + for { + queue, userID, idx := q.queues.getNextQueueForQuerier(last.last, querierID) + last.last = idx + if queue == nil { + break + } + + // Pick next request from the queue. + for { + request := <-queue + if len(queue) == 0 { + q.queues.deleteQueue(userID) + } + + q.queueLength.WithLabelValues(userID).Dec() + + // Tell close() we've processed a request. + q.cond.Broadcast() + + return request, last, nil + } + } + + // There are no unexpired requests, so we can get back + // and wait for more requests. + querierWait = true + goto FindQueue +} + +func (q *RequestQueue) Stop() { + q.mtx.Lock() + defer q.mtx.Unlock() + + for q.queues.len() > 0 && q.connectedQuerierWorkers.Load() > 0 { + q.cond.Wait() + } + + // Only stop after dispatching enqueued requests. + q.stopped = true + + // If there are still goroutines in GetNextRequestForQuerier method, they get notified. + q.cond.Broadcast() +} + +func (q *RequestQueue) RegisterQuerierConnection(querier string) { + q.connectedQuerierWorkers.Inc() + + q.mtx.Lock() + defer q.mtx.Unlock() + q.queues.addQuerierConnection(querier) +} + +func (q *RequestQueue) UnregisterQuerierConnection(querier string) { + q.connectedQuerierWorkers.Dec() + + q.mtx.Lock() + defer q.mtx.Unlock() + q.queues.removeQuerierConnection(querier) +} + +// When querier is waiting for next request, this unblocks the method. +func (q *RequestQueue) QuerierDisconnecting() { + q.cond.Broadcast() +} + +func (q *RequestQueue) GetConnectedQuerierWorkersMetric() float64 { + return float64(q.connectedQuerierWorkers.Load()) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend_querier_queues.go b/vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/user_queues.go similarity index 97% rename from vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend_querier_queues.go rename to vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/user_queues.go index 23c1435d4379..223a7500ff4c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/querier/frontend/frontend_querier_queues.go +++ b/vendor/github.com/cortexproject/cortex/pkg/scheduler/queue/user_queues.go @@ -1,4 +1,4 @@ -package frontend +package queue import ( "math/rand" @@ -26,7 +26,7 @@ type queues struct { } type userQueue struct { - ch chan *request + ch chan Request // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. @@ -74,7 +74,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan *request { +func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -88,7 +88,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan *request { if uq == nil { uq = &userQueue{ - ch: make(chan *request, q.maxUserQueueSize), + ch: make(chan Request, q.maxUserQueueSize), seed: util.ShuffleShardSeed(userID, ""), index: -1, } @@ -121,7 +121,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan *request { // Finds next queue for the querier. To support fair scheduling between users, client is expected // to pass last user index returned by this function as argument. Is there was no previous // last user index, use -1. -func (q *queues) getNextQueueForQuerier(lastUserIndex int, querier string) (chan *request, string, int) { +func (q *queues) getNextQueueForQuerier(lastUserIndex int, querier string) (chan Request, string, int) { uid := lastUserIndex for iters := 0; iters < len(q.users); iters++ { diff --git a/vendor/github.com/cortexproject/cortex/pkg/scheduler/scheduler.go b/vendor/github.com/cortexproject/cortex/pkg/scheduler/scheduler.go new file mode 100644 index 000000000000..987f6866d0a4 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/scheduler/scheduler.go @@ -0,0 +1,455 @@ +package scheduler + +import ( + "context" + "errors" + "flag" + "net/http" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + otgrpc "github.com/opentracing-contrib/go-grpc" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/middleware" + "github.com/weaveworks/common/user" + "google.golang.org/grpc" + + "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" + "github.com/cortexproject/cortex/pkg/scheduler/queue" + "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" + "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/grpcutil" + "github.com/cortexproject/cortex/pkg/util/services" +) + +var ( + errSchedulerIsNotRunning = errors.New("scheduler is not running") +) + +// Scheduler is responsible for queueing and dispatching queries to Queriers. +type Scheduler struct { + services.Service + + cfg Config + log log.Logger + + limits Limits + + connectedFrontendsMu sync.Mutex + connectedFrontends map[string]*connectedFrontend + + requestQueue *queue.RequestQueue + + pendingRequestsMu sync.Mutex + pendingRequests map[requestKey]*schedulerRequest // Request is kept in this map even after being dispatched to querier. It can still be canceled at that time. + + // Metrics. + connectedQuerierClients prometheus.GaugeFunc + connectedFrontendClients prometheus.GaugeFunc + queueDuration prometheus.Histogram +} + +type requestKey struct { + frontendAddr string + queryID uint64 +} + +type connectedFrontend struct { + connections int + + // This context is used for running all queries from the same frontend. + // When last frontend connection is closed, context is canceled. + ctx context.Context + cancel context.CancelFunc +} + +type Config struct { + MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant"` + + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config" doc:"description=This configures the gRPC client used to report errors back to the query-frontend."` +} + +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.IntVar(&cfg.MaxOutstandingPerTenant, "query-scheduler.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per query-scheduler. In-flight requests above this limit will fail with HTTP response status code 429.") + cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-scheduler.grpc-client-config", f) +} + +// NewScheduler creates a new Scheduler. +func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer) (*Scheduler, error) { + queueLength := promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_query_scheduler_queue_length", + Help: "Number of queries in the queue.", + }, []string{"user"}) + + s := &Scheduler{ + cfg: cfg, + log: log, + limits: limits, + + requestQueue: queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, queueLength), + pendingRequests: map[requestKey]*schedulerRequest{}, + connectedFrontends: map[string]*connectedFrontend{}, + } + + s.queueDuration = promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_query_scheduler_queue_duration_seconds", + Help: "Time spend by requests in queue before getting picked up by a querier.", + Buckets: prometheus.DefBuckets, + }) + s.connectedQuerierClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_query_scheduler_connected_querier_clients", + Help: "Number of querier worker clients currently connected to the query-scheduler.", + }, s.requestQueue.GetConnectedQuerierWorkersMetric) + s.connectedFrontendClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_query_scheduler_connected_frontend_clients", + Help: "Number of query-frontend worker clients currently connected to the query-scheduler.", + }, s.getConnectedFrontendClientsMetric) + + s.Service = services.NewIdleService(nil, s.stopping) + return s, nil +} + +// Limits needed for the Query Frontend - interface used for decoupling. +type Limits interface { + // Returns max queriers to use per tenant, or 0 if shuffle sharding is disabled. + MaxQueriersPerUser(user string) int +} + +type schedulerRequest struct { + frontendAddress string + userID string + queryID uint64 + request *httpgrpc.HTTPRequest + + enqueueTime time.Time + + ctx context.Context + ctxCancel context.CancelFunc + queueSpan opentracing.Span + + // This is only used for testing. + parentSpanContext opentracing.SpanContext +} + +// This method handles connection from frontend. +func (s *Scheduler) FrontendLoop(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) error { + frontendAddress, frontendCtx, err := s.frontendConnected(frontend) + if err != nil { + return err + } + defer s.frontendDisconnected(frontendAddress) + + // Response to INIT. If scheduler is not running, we skip for-loop, send SHUTTING_DOWN and exit this method. + if s.State() == services.Running { + if err := frontend.Send(&schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}); err != nil { + return err + } + } + + // We stop accepting new queries in Stopping state. By returning quickly, we disconnect frontends, which in turns + // cancels all their queries. + for s.State() == services.Running { + msg, err := frontend.Recv() + if err != nil { + return err + } + + if s.State() != services.Running { + break // break out of the loop, and send SHUTTING_DOWN message. + } + + var resp *schedulerpb.SchedulerToFrontend + + switch msg.GetType() { + case schedulerpb.ENQUEUE: + err = s.enqueueRequest(frontendCtx, frontendAddress, msg) + switch { + case err == nil: + resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} + case err == queue.ErrTooManyRequests: + resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.TOO_MANY_REQUESTS_PER_TENANT} + default: + resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.ERROR, Error: err.Error()} + } + + case schedulerpb.CANCEL: + s.cancelRequestAndRemoveFromPending(frontendAddress, msg.QueryID) + resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} + + default: + level.Error(s.log).Log("msg", "unknown request type from frontend", "addr", frontendAddress, "type", msg.GetType()) + return errors.New("unknown request type") + } + + err = frontend.Send(resp) + // Failure to send response results in ending this connection. + if err != nil { + return err + } + } + + // Report shutdown back to frontend, so that it can retry with different scheduler. Also stop the frontend loop. + return frontend.Send(&schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}) +} + +func (s *Scheduler) frontendConnected(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) (string, context.Context, error) { + msg, err := frontend.Recv() + if err != nil { + return "", nil, err + } + if msg.Type != schedulerpb.INIT || msg.FrontendAddress == "" { + return "", nil, errors.New("no frontend address") + } + + s.connectedFrontendsMu.Lock() + defer s.connectedFrontendsMu.Unlock() + + cf := s.connectedFrontends[msg.FrontendAddress] + if cf == nil { + cf = &connectedFrontend{ + connections: 0, + } + cf.ctx, cf.cancel = context.WithCancel(context.Background()) + s.connectedFrontends[msg.FrontendAddress] = cf + } + + cf.connections++ + return msg.FrontendAddress, cf.ctx, nil +} + +func (s *Scheduler) frontendDisconnected(frontendAddress string) { + s.connectedFrontendsMu.Lock() + defer s.connectedFrontendsMu.Unlock() + + cf := s.connectedFrontends[frontendAddress] + cf.connections-- + if cf.connections == 0 { + delete(s.connectedFrontends, frontendAddress) + cf.cancel() + } +} + +func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr string, msg *schedulerpb.FrontendToScheduler) error { + // Create new context for this request, to support cancellation. + ctx, cancel := context.WithCancel(frontendContext) + shouldCancel := true + defer func() { + if shouldCancel { + cancel() + } + }() + + // Extract tracing information from headers in HTTP request. FrontendContext doesn't have the correct tracing + // information, since that is a long-running request. + tracer := opentracing.GlobalTracer() + parentSpanContext, err := grpcutil.GetParentSpanForRequest(tracer, msg.HttpRequest) + if err != nil { + return err + } + + userID := msg.GetUserID() + + req := &schedulerRequest{ + frontendAddress: frontendAddr, + userID: msg.UserID, + queryID: msg.QueryID, + request: msg.HttpRequest, + } + + req.parentSpanContext = parentSpanContext + req.queueSpan, req.ctx = opentracing.StartSpanFromContextWithTracer(ctx, tracer, "queued", opentracing.ChildOf(parentSpanContext)) + req.enqueueTime = time.Now() + req.ctxCancel = cancel + + maxQueriers := s.limits.MaxQueriersPerUser(userID) + + return s.requestQueue.EnqueueRequest(userID, req, maxQueriers, func() { + shouldCancel = false + + s.pendingRequestsMu.Lock() + defer s.pendingRequestsMu.Unlock() + s.pendingRequests[requestKey{frontendAddr: frontendAddr, queryID: msg.QueryID}] = req + }) +} + +// This method doesn't do removal from the queue. +func (s *Scheduler) cancelRequestAndRemoveFromPending(frontendAddr string, queryID uint64) { + s.pendingRequestsMu.Lock() + defer s.pendingRequestsMu.Unlock() + + key := requestKey{frontendAddr: frontendAddr, queryID: queryID} + req := s.pendingRequests[key] + if req != nil { + req.ctxCancel() + } + delete(s.pendingRequests, key) +} + +// QuerierLoop is started by querier to receive queries from scheduler. +func (s *Scheduler) QuerierLoop(querier schedulerpb.SchedulerForQuerier_QuerierLoopServer) error { + resp, err := querier.Recv() + if err != nil { + return err + } + + querierID := resp.GetQuerierID() + + s.requestQueue.RegisterQuerierConnection(querierID) + defer s.requestQueue.UnregisterQuerierConnection(querierID) + + // If the downstream connection to querier is cancelled, + // we need to ping the condition variable to unblock getNextRequestForQuerier. + // Ideally we'd have ctx aware condition variables... + go func() { + <-querier.Context().Done() + s.requestQueue.QuerierDisconnecting() + }() + + lastUserIndex := queue.FirstUser() + + // In stopping state scheduler is not accepting new queries, but still dispatching queries in the queues. + for s.isRunningOrStopping() { + req, idx, err := s.requestQueue.GetNextRequestForQuerier(querier.Context(), lastUserIndex, querierID) + if err != nil { + return err + } + lastUserIndex = idx + + r := req.(*schedulerRequest) + + s.queueDuration.Observe(time.Since(r.enqueueTime).Seconds()) + r.queueSpan.Finish() + + /* + We want to dequeue the next unexpired request from the chosen tenant queue. + The chance of choosing a particular tenant for dequeueing is (1/active_tenants). + This is problematic under load, especially with other middleware enabled such as + querier.split-by-interval, where one request may fan out into many. + If expired requests aren't exhausted before checking another tenant, it would take + n_active_tenants * n_expired_requests_at_front_of_queue requests being processed + before an active request was handled for the tenant in question. + If this tenant meanwhile continued to queue requests, + it's possible that it's own queue would perpetually contain only expired requests. + */ + + if r.ctx.Err() != nil { + // Remove from pending requests. + s.cancelRequestAndRemoveFromPending(r.frontendAddress, r.queryID) + + lastUserIndex = lastUserIndex.ReuseLastUser() + continue + } + + if err := s.forwardRequestToQuerier(querier, r); err != nil { + return err + } + } + + return errSchedulerIsNotRunning +} + +func (s *Scheduler) forwardRequestToQuerier(querier schedulerpb.SchedulerForQuerier_QuerierLoopServer, req *schedulerRequest) error { + // Make sure to cancel request at the end to cleanup resources. + defer s.cancelRequestAndRemoveFromPending(req.frontendAddress, req.queryID) + + // Handle the stream sending & receiving on a goroutine so we can + // monitoring the contexts in a select and cancel things appropriately. + errCh := make(chan error, 1) + go func() { + err := querier.Send(&schedulerpb.SchedulerToQuerier{ + UserID: req.userID, + QueryID: req.queryID, + FrontendAddress: req.frontendAddress, + HttpRequest: req.request, + }) + if err != nil { + errCh <- err + return + } + + _, err = querier.Recv() + errCh <- err + }() + + select { + case <-req.ctx.Done(): + // If the upstream request is cancelled (eg. frontend issued CANCEL or closed connection), + // we need to cancel the downstream req. Only way we can do that is to close the stream (by returning error here). + // Querier is expecting this semantics. + return req.ctx.Err() + + case err := <-errCh: + // Is there was an error handling this request due to network IO, + // then error out this upstream request _and_ stream. + + if err != nil { + s.forwardErrorToFrontend(req.ctx, req, err) + } + return err + } +} + +func (s *Scheduler) forwardErrorToFrontend(ctx context.Context, req *schedulerRequest, requestErr error) { + opts, err := s.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{ + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), + middleware.ClientUserHeaderInterceptor}, + nil) + if err != nil { + level.Warn(s.log).Log("msg", "failed to create gRPC options for the connection to frontend to report error", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) + return + } + + conn, err := grpc.DialContext(ctx, req.frontendAddress, opts...) + if err != nil { + level.Warn(s.log).Log("msg", "failed to create gRPC connection to frontend to report error", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) + return + } + + defer func() { + _ = conn.Close() + }() + + client := frontendv2pb.NewFrontendForQuerierClient(conn) + + userCtx := user.InjectOrgID(ctx, req.userID) + _, err = client.QueryResult(userCtx, &frontendv2pb.QueryResultRequest{ + QueryID: req.queryID, + HttpResponse: &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(requestErr.Error()), + }, + }) + + if err != nil { + level.Warn(s.log).Log("msg", "failed to forward error to frontend", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) + return + } +} + +func (s *Scheduler) isRunningOrStopping() bool { + st := s.State() + return st == services.Running || st == services.Stopping +} + +// Close the Scheduler. +func (s *Scheduler) stopping(_ error) error { + s.requestQueue.Stop() + return nil +} + +func (s *Scheduler) getConnectedFrontendClientsMetric() float64 { + s.connectedFrontendsMu.Lock() + defer s.connectedFrontendsMu.Unlock() + + count := 0 + for _, workers := range s.connectedFrontends { + count += workers.connections + } + + return float64(count) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.pb.go b/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.pb.go new file mode 100644 index 000000000000..a2698e5749a9 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.pb.go @@ -0,0 +1,1809 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: scheduler.proto + +package schedulerpb + +import ( + context "context" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + httpgrpc "github.com/weaveworks/common/httpgrpc" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strconv "strconv" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type FrontendToSchedulerType int32 + +const ( + INIT FrontendToSchedulerType = 0 + ENQUEUE FrontendToSchedulerType = 1 + CANCEL FrontendToSchedulerType = 2 +) + +var FrontendToSchedulerType_name = map[int32]string{ + 0: "INIT", + 1: "ENQUEUE", + 2: "CANCEL", +} + +var FrontendToSchedulerType_value = map[string]int32{ + "INIT": 0, + "ENQUEUE": 1, + "CANCEL": 2, +} + +func (FrontendToSchedulerType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{0} +} + +type SchedulerToFrontendStatus int32 + +const ( + OK SchedulerToFrontendStatus = 0 + TOO_MANY_REQUESTS_PER_TENANT SchedulerToFrontendStatus = 1 + ERROR SchedulerToFrontendStatus = 2 + SHUTTING_DOWN SchedulerToFrontendStatus = 3 +) + +var SchedulerToFrontendStatus_name = map[int32]string{ + 0: "OK", + 1: "TOO_MANY_REQUESTS_PER_TENANT", + 2: "ERROR", + 3: "SHUTTING_DOWN", +} + +var SchedulerToFrontendStatus_value = map[string]int32{ + "OK": 0, + "TOO_MANY_REQUESTS_PER_TENANT": 1, + "ERROR": 2, + "SHUTTING_DOWN": 3, +} + +func (SchedulerToFrontendStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{1} +} + +// Querier reports its own clientID when it connects, so that scheduler knows how many *different* queriers are connected. +// To signal that querier is ready to accept another request, querier sends empty message. +type QuerierToScheduler struct { + QuerierID string `protobuf:"bytes,1,opt,name=querierID,proto3" json:"querierID,omitempty"` +} + +func (m *QuerierToScheduler) Reset() { *m = QuerierToScheduler{} } +func (*QuerierToScheduler) ProtoMessage() {} +func (*QuerierToScheduler) Descriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{0} +} +func (m *QuerierToScheduler) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *QuerierToScheduler) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_QuerierToScheduler.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *QuerierToScheduler) XXX_Merge(src proto.Message) { + xxx_messageInfo_QuerierToScheduler.Merge(m, src) +} +func (m *QuerierToScheduler) XXX_Size() int { + return m.Size() +} +func (m *QuerierToScheduler) XXX_DiscardUnknown() { + xxx_messageInfo_QuerierToScheduler.DiscardUnknown(m) +} + +var xxx_messageInfo_QuerierToScheduler proto.InternalMessageInfo + +func (m *QuerierToScheduler) GetQuerierID() string { + if m != nil { + return m.QuerierID + } + return "" +} + +type SchedulerToQuerier struct { + // Query ID as reported by frontend. When querier sends the response back to frontend (using frontendAddress), + // it identifies the query by using this ID. + QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` + HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,2,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` + // Where should querier send HTTP Response to (using FrontendForQuerier interface). + FrontendAddress string `protobuf:"bytes,3,opt,name=frontendAddress,proto3" json:"frontendAddress,omitempty"` + // User who initiated the request. Needed to send reply back to frontend. + UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` +} + +func (m *SchedulerToQuerier) Reset() { *m = SchedulerToQuerier{} } +func (*SchedulerToQuerier) ProtoMessage() {} +func (*SchedulerToQuerier) Descriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{1} +} +func (m *SchedulerToQuerier) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SchedulerToQuerier) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SchedulerToQuerier.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SchedulerToQuerier) XXX_Merge(src proto.Message) { + xxx_messageInfo_SchedulerToQuerier.Merge(m, src) +} +func (m *SchedulerToQuerier) XXX_Size() int { + return m.Size() +} +func (m *SchedulerToQuerier) XXX_DiscardUnknown() { + xxx_messageInfo_SchedulerToQuerier.DiscardUnknown(m) +} + +var xxx_messageInfo_SchedulerToQuerier proto.InternalMessageInfo + +func (m *SchedulerToQuerier) GetQueryID() uint64 { + if m != nil { + return m.QueryID + } + return 0 +} + +func (m *SchedulerToQuerier) GetHttpRequest() *httpgrpc.HTTPRequest { + if m != nil { + return m.HttpRequest + } + return nil +} + +func (m *SchedulerToQuerier) GetFrontendAddress() string { + if m != nil { + return m.FrontendAddress + } + return "" +} + +func (m *SchedulerToQuerier) GetUserID() string { + if m != nil { + return m.UserID + } + return "" +} + +type FrontendToScheduler struct { + Type FrontendToSchedulerType `protobuf:"varint,1,opt,name=type,proto3,enum=schedulerpb.FrontendToSchedulerType" json:"type,omitempty"` + // Used by INIT message. Will be put into all requests passed to querier. + FrontendAddress string `protobuf:"bytes,2,opt,name=frontendAddress,proto3" json:"frontendAddress,omitempty"` + // Used by ENQUEUE and CANCEL. + // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. + QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"` + // Following are used by ENQUEUE only. + UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` + HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` +} + +func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } +func (*FrontendToScheduler) ProtoMessage() {} +func (*FrontendToScheduler) Descriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{2} +} +func (m *FrontendToScheduler) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FrontendToScheduler) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FrontendToScheduler.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FrontendToScheduler) XXX_Merge(src proto.Message) { + xxx_messageInfo_FrontendToScheduler.Merge(m, src) +} +func (m *FrontendToScheduler) XXX_Size() int { + return m.Size() +} +func (m *FrontendToScheduler) XXX_DiscardUnknown() { + xxx_messageInfo_FrontendToScheduler.DiscardUnknown(m) +} + +var xxx_messageInfo_FrontendToScheduler proto.InternalMessageInfo + +func (m *FrontendToScheduler) GetType() FrontendToSchedulerType { + if m != nil { + return m.Type + } + return INIT +} + +func (m *FrontendToScheduler) GetFrontendAddress() string { + if m != nil { + return m.FrontendAddress + } + return "" +} + +func (m *FrontendToScheduler) GetQueryID() uint64 { + if m != nil { + return m.QueryID + } + return 0 +} + +func (m *FrontendToScheduler) GetUserID() string { + if m != nil { + return m.UserID + } + return "" +} + +func (m *FrontendToScheduler) GetHttpRequest() *httpgrpc.HTTPRequest { + if m != nil { + return m.HttpRequest + } + return nil +} + +type SchedulerToFrontend struct { + Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` +} + +func (m *SchedulerToFrontend) Reset() { *m = SchedulerToFrontend{} } +func (*SchedulerToFrontend) ProtoMessage() {} +func (*SchedulerToFrontend) Descriptor() ([]byte, []int) { + return fileDescriptor_2b3fc28395a6d9c5, []int{3} +} +func (m *SchedulerToFrontend) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SchedulerToFrontend) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SchedulerToFrontend.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SchedulerToFrontend) XXX_Merge(src proto.Message) { + xxx_messageInfo_SchedulerToFrontend.Merge(m, src) +} +func (m *SchedulerToFrontend) XXX_Size() int { + return m.Size() +} +func (m *SchedulerToFrontend) XXX_DiscardUnknown() { + xxx_messageInfo_SchedulerToFrontend.DiscardUnknown(m) +} + +var xxx_messageInfo_SchedulerToFrontend proto.InternalMessageInfo + +func (m *SchedulerToFrontend) GetStatus() SchedulerToFrontendStatus { + if m != nil { + return m.Status + } + return OK +} + +func (m *SchedulerToFrontend) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func init() { + proto.RegisterEnum("schedulerpb.FrontendToSchedulerType", FrontendToSchedulerType_name, FrontendToSchedulerType_value) + proto.RegisterEnum("schedulerpb.SchedulerToFrontendStatus", SchedulerToFrontendStatus_name, SchedulerToFrontendStatus_value) + proto.RegisterType((*QuerierToScheduler)(nil), "schedulerpb.QuerierToScheduler") + proto.RegisterType((*SchedulerToQuerier)(nil), "schedulerpb.SchedulerToQuerier") + proto.RegisterType((*FrontendToScheduler)(nil), "schedulerpb.FrontendToScheduler") + proto.RegisterType((*SchedulerToFrontend)(nil), "schedulerpb.SchedulerToFrontend") +} + +func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } + +var fileDescriptor_2b3fc28395a6d9c5 = []byte{ + // 570 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x5d, 0x6f, 0x12, 0x41, + 0x14, 0xdd, 0xa1, 0x40, 0xe5, 0xa2, 0x76, 0x9d, 0x56, 0x45, 0xd2, 0x4c, 0x09, 0x31, 0x86, 0x34, + 0x11, 0x0c, 0x9a, 0xe8, 0x83, 0x31, 0xc1, 0x76, 0x6b, 0x89, 0x75, 0x29, 0xc3, 0x10, 0x3f, 0x5e, + 0x48, 0x81, 0x29, 0x34, 0x2d, 0xcc, 0x76, 0x76, 0xd7, 0x86, 0x37, 0x7f, 0x82, 0x3f, 0x43, 0xff, + 0x89, 0x8f, 0x3c, 0xf6, 0x51, 0x16, 0x1f, 0x7c, 0xec, 0x4f, 0x30, 0x1d, 0x96, 0x75, 0xa9, 0x90, + 0xfa, 0x76, 0xef, 0xdd, 0x73, 0x72, 0xce, 0x3d, 0x33, 0x3b, 0xb0, 0x62, 0xb7, 0xba, 0xbc, 0xed, + 0x9e, 0x70, 0x99, 0xb7, 0xa4, 0x70, 0x04, 0x4e, 0x06, 0x03, 0xab, 0x99, 0x7e, 0xdc, 0x39, 0x72, + 0xba, 0x6e, 0x33, 0xdf, 0x12, 0xbd, 0x42, 0x47, 0x74, 0x44, 0x41, 0x61, 0x9a, 0xee, 0xa1, 0xea, + 0x54, 0xa3, 0xaa, 0x09, 0x37, 0xfd, 0x2c, 0x04, 0x3f, 0xe3, 0x07, 0x9f, 0xf9, 0x99, 0x90, 0xc7, + 0x76, 0xa1, 0x25, 0x7a, 0x3d, 0xd1, 0x2f, 0x74, 0x1d, 0xc7, 0xea, 0x48, 0xab, 0x15, 0x14, 0x13, + 0x56, 0xb6, 0x08, 0xb8, 0xea, 0x72, 0x79, 0xc4, 0x25, 0x13, 0xb5, 0xa9, 0x38, 0x5e, 0x87, 0xc4, + 0xe9, 0x64, 0x5a, 0xde, 0x4e, 0xa1, 0x0c, 0xca, 0x25, 0xe8, 0xdf, 0x41, 0xf6, 0x3b, 0x02, 0x1c, + 0x60, 0x99, 0xf0, 0xf9, 0x38, 0x05, 0xcb, 0x97, 0x98, 0x81, 0x4f, 0x89, 0xd2, 0x69, 0x8b, 0x9f, + 0x43, 0xf2, 0x52, 0x96, 0xf2, 0x53, 0x97, 0xdb, 0x4e, 0x2a, 0x92, 0x41, 0xb9, 0x64, 0xf1, 0x6e, + 0x3e, 0xb0, 0xb2, 0xcb, 0xd8, 0xbe, 0xff, 0x91, 0x86, 0x91, 0x38, 0x07, 0x2b, 0x87, 0x52, 0xf4, + 0x1d, 0xde, 0x6f, 0x97, 0xda, 0x6d, 0xc9, 0x6d, 0x3b, 0xb5, 0xa4, 0xdc, 0x5c, 0x1d, 0xe3, 0x7b, + 0x10, 0x77, 0x6d, 0x65, 0x37, 0xaa, 0x00, 0x7e, 0x97, 0xfd, 0x85, 0x60, 0x75, 0xc7, 0xc7, 0x86, + 0x37, 0x7c, 0x01, 0x51, 0x67, 0x60, 0x71, 0xe5, 0xf4, 0x76, 0xf1, 0x61, 0x3e, 0x14, 0x7c, 0x7e, + 0x0e, 0x9e, 0x0d, 0x2c, 0x4e, 0x15, 0x63, 0x9e, 0xa7, 0xc8, 0x7c, 0x4f, 0xa1, 0x40, 0x96, 0x66, + 0x03, 0x59, 0xe0, 0xf6, 0x6a, 0x50, 0xb1, 0xff, 0x0d, 0x2a, 0x7b, 0x0c, 0xab, 0xa1, 0x13, 0x99, + 0x2e, 0x80, 0x5f, 0x41, 0xdc, 0x76, 0x0e, 0x1c, 0xd7, 0xf6, 0xf7, 0x7c, 0x34, 0xb3, 0xe7, 0x1c, + 0x46, 0x4d, 0xa1, 0xa9, 0xcf, 0xc2, 0x6b, 0x10, 0xe3, 0x52, 0x0a, 0xe9, 0x6f, 0x38, 0x69, 0x36, + 0x5f, 0xc2, 0xfd, 0x05, 0x11, 0xe1, 0x1b, 0x10, 0x2d, 0x9b, 0x65, 0xa6, 0x6b, 0x38, 0x09, 0xcb, + 0x86, 0x59, 0xad, 0x1b, 0x75, 0x43, 0x47, 0x18, 0x20, 0xbe, 0x55, 0x32, 0xb7, 0x8c, 0x3d, 0x3d, + 0xb2, 0xd9, 0x82, 0x07, 0x0b, 0x85, 0x71, 0x1c, 0x22, 0x95, 0xb7, 0xba, 0x86, 0x33, 0xb0, 0xce, + 0x2a, 0x95, 0xc6, 0xbb, 0x92, 0xf9, 0xb1, 0x41, 0x8d, 0x6a, 0xdd, 0xa8, 0xb1, 0x5a, 0x63, 0xdf, + 0xa0, 0x0d, 0x66, 0x98, 0x25, 0x93, 0xe9, 0x08, 0x27, 0x20, 0x66, 0x50, 0x5a, 0xa1, 0x7a, 0x04, + 0xdf, 0x81, 0x5b, 0xb5, 0xdd, 0x3a, 0x63, 0x65, 0xf3, 0x4d, 0x63, 0xbb, 0xf2, 0xde, 0xd4, 0x97, + 0x8a, 0x27, 0xa1, 0x3c, 0x76, 0x84, 0x9c, 0x5e, 0xd1, 0x3a, 0x24, 0xfd, 0x72, 0x4f, 0x08, 0x0b, + 0x6f, 0xcc, 0xc4, 0xf1, 0xef, 0x7f, 0x90, 0xde, 0x58, 0x94, 0x97, 0x8f, 0xcd, 0x6a, 0x39, 0xf4, + 0x04, 0x15, 0x2d, 0x58, 0x0b, 0xab, 0x05, 0xf1, 0x7f, 0x80, 0x9b, 0xd3, 0x5a, 0xe9, 0x65, 0xae, + 0xbb, 0x66, 0xe9, 0xcc, 0x75, 0x07, 0x34, 0x51, 0x7c, 0x5d, 0x1a, 0x8e, 0x88, 0x76, 0x3e, 0x22, + 0xda, 0xc5, 0x88, 0xa0, 0x2f, 0x1e, 0x41, 0xdf, 0x3c, 0x82, 0x7e, 0x78, 0x04, 0x0d, 0x3d, 0x82, + 0x7e, 0x7a, 0x04, 0xfd, 0xf6, 0x88, 0x76, 0xe1, 0x11, 0xf4, 0x75, 0x4c, 0xb4, 0xe1, 0x98, 0x68, + 0xe7, 0x63, 0xa2, 0x7d, 0x0a, 0x3f, 0x2f, 0xcd, 0xb8, 0x7a, 0x00, 0x9e, 0xfe, 0x09, 0x00, 0x00, + 0xff, 0xff, 0x89, 0xbf, 0xda, 0x9a, 0x85, 0x04, 0x00, 0x00, +} + +func (x FrontendToSchedulerType) String() string { + s, ok := FrontendToSchedulerType_name[int32(x)] + if ok { + return s + } + return strconv.Itoa(int(x)) +} +func (x SchedulerToFrontendStatus) String() string { + s, ok := SchedulerToFrontendStatus_name[int32(x)] + if ok { + return s + } + return strconv.Itoa(int(x)) +} +func (this *QuerierToScheduler) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*QuerierToScheduler) + if !ok { + that2, ok := that.(QuerierToScheduler) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.QuerierID != that1.QuerierID { + return false + } + return true +} +func (this *SchedulerToQuerier) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*SchedulerToQuerier) + if !ok { + that2, ok := that.(SchedulerToQuerier) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.QueryID != that1.QueryID { + return false + } + if !this.HttpRequest.Equal(that1.HttpRequest) { + return false + } + if this.FrontendAddress != that1.FrontendAddress { + return false + } + if this.UserID != that1.UserID { + return false + } + return true +} +func (this *FrontendToScheduler) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*FrontendToScheduler) + if !ok { + that2, ok := that.(FrontendToScheduler) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Type != that1.Type { + return false + } + if this.FrontendAddress != that1.FrontendAddress { + return false + } + if this.QueryID != that1.QueryID { + return false + } + if this.UserID != that1.UserID { + return false + } + if !this.HttpRequest.Equal(that1.HttpRequest) { + return false + } + return true +} +func (this *SchedulerToFrontend) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*SchedulerToFrontend) + if !ok { + that2, ok := that.(SchedulerToFrontend) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Status != that1.Status { + return false + } + if this.Error != that1.Error { + return false + } + return true +} +func (this *QuerierToScheduler) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&schedulerpb.QuerierToScheduler{") + s = append(s, "QuerierID: "+fmt.Sprintf("%#v", this.QuerierID)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *SchedulerToQuerier) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 8) + s = append(s, "&schedulerpb.SchedulerToQuerier{") + s = append(s, "QueryID: "+fmt.Sprintf("%#v", this.QueryID)+",\n") + if this.HttpRequest != nil { + s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") + } + s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") + s = append(s, "UserID: "+fmt.Sprintf("%#v", this.UserID)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *FrontendToScheduler) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 9) + s = append(s, "&schedulerpb.FrontendToScheduler{") + s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n") + s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") + s = append(s, "QueryID: "+fmt.Sprintf("%#v", this.QueryID)+",\n") + s = append(s, "UserID: "+fmt.Sprintf("%#v", this.UserID)+",\n") + if this.HttpRequest != nil { + s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *SchedulerToFrontend) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&schedulerpb.SchedulerToFrontend{") + s = append(s, "Status: "+fmt.Sprintf("%#v", this.Status)+",\n") + s = append(s, "Error: "+fmt.Sprintf("%#v", this.Error)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringScheduler(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// SchedulerForQuerierClient is the client API for SchedulerForQuerier service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SchedulerForQuerierClient interface { + // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for + // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, + // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. + // + // Long-running loop is used to detect broken connection between scheduler and querier. This is important + // for scheduler to keep a list of connected queriers up-to-date. + QuerierLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForQuerier_QuerierLoopClient, error) +} + +type schedulerForQuerierClient struct { + cc *grpc.ClientConn +} + +func NewSchedulerForQuerierClient(cc *grpc.ClientConn) SchedulerForQuerierClient { + return &schedulerForQuerierClient{cc} +} + +func (c *schedulerForQuerierClient) QuerierLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForQuerier_QuerierLoopClient, error) { + stream, err := c.cc.NewStream(ctx, &_SchedulerForQuerier_serviceDesc.Streams[0], "/schedulerpb.SchedulerForQuerier/QuerierLoop", opts...) + if err != nil { + return nil, err + } + x := &schedulerForQuerierQuerierLoopClient{stream} + return x, nil +} + +type SchedulerForQuerier_QuerierLoopClient interface { + Send(*QuerierToScheduler) error + Recv() (*SchedulerToQuerier, error) + grpc.ClientStream +} + +type schedulerForQuerierQuerierLoopClient struct { + grpc.ClientStream +} + +func (x *schedulerForQuerierQuerierLoopClient) Send(m *QuerierToScheduler) error { + return x.ClientStream.SendMsg(m) +} + +func (x *schedulerForQuerierQuerierLoopClient) Recv() (*SchedulerToQuerier, error) { + m := new(SchedulerToQuerier) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// SchedulerForQuerierServer is the server API for SchedulerForQuerier service. +type SchedulerForQuerierServer interface { + // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for + // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, + // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. + // + // Long-running loop is used to detect broken connection between scheduler and querier. This is important + // for scheduler to keep a list of connected queriers up-to-date. + QuerierLoop(SchedulerForQuerier_QuerierLoopServer) error +} + +// UnimplementedSchedulerForQuerierServer can be embedded to have forward compatible implementations. +type UnimplementedSchedulerForQuerierServer struct { +} + +func (*UnimplementedSchedulerForQuerierServer) QuerierLoop(srv SchedulerForQuerier_QuerierLoopServer) error { + return status.Errorf(codes.Unimplemented, "method QuerierLoop not implemented") +} + +func RegisterSchedulerForQuerierServer(s *grpc.Server, srv SchedulerForQuerierServer) { + s.RegisterService(&_SchedulerForQuerier_serviceDesc, srv) +} + +func _SchedulerForQuerier_QuerierLoop_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SchedulerForQuerierServer).QuerierLoop(&schedulerForQuerierQuerierLoopServer{stream}) +} + +type SchedulerForQuerier_QuerierLoopServer interface { + Send(*SchedulerToQuerier) error + Recv() (*QuerierToScheduler, error) + grpc.ServerStream +} + +type schedulerForQuerierQuerierLoopServer struct { + grpc.ServerStream +} + +func (x *schedulerForQuerierQuerierLoopServer) Send(m *SchedulerToQuerier) error { + return x.ServerStream.SendMsg(m) +} + +func (x *schedulerForQuerierQuerierLoopServer) Recv() (*QuerierToScheduler, error) { + m := new(QuerierToScheduler) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _SchedulerForQuerier_serviceDesc = grpc.ServiceDesc{ + ServiceName: "schedulerpb.SchedulerForQuerier", + HandlerType: (*SchedulerForQuerierServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "QuerierLoop", + Handler: _SchedulerForQuerier_QuerierLoop_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "scheduler.proto", +} + +// SchedulerForFrontendClient is the client API for SchedulerForFrontend service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SchedulerForFrontendClient interface { + // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and + // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. + // + // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both + // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending + // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. + FrontendLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForFrontend_FrontendLoopClient, error) +} + +type schedulerForFrontendClient struct { + cc *grpc.ClientConn +} + +func NewSchedulerForFrontendClient(cc *grpc.ClientConn) SchedulerForFrontendClient { + return &schedulerForFrontendClient{cc} +} + +func (c *schedulerForFrontendClient) FrontendLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForFrontend_FrontendLoopClient, error) { + stream, err := c.cc.NewStream(ctx, &_SchedulerForFrontend_serviceDesc.Streams[0], "/schedulerpb.SchedulerForFrontend/FrontendLoop", opts...) + if err != nil { + return nil, err + } + x := &schedulerForFrontendFrontendLoopClient{stream} + return x, nil +} + +type SchedulerForFrontend_FrontendLoopClient interface { + Send(*FrontendToScheduler) error + Recv() (*SchedulerToFrontend, error) + grpc.ClientStream +} + +type schedulerForFrontendFrontendLoopClient struct { + grpc.ClientStream +} + +func (x *schedulerForFrontendFrontendLoopClient) Send(m *FrontendToScheduler) error { + return x.ClientStream.SendMsg(m) +} + +func (x *schedulerForFrontendFrontendLoopClient) Recv() (*SchedulerToFrontend, error) { + m := new(SchedulerToFrontend) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// SchedulerForFrontendServer is the server API for SchedulerForFrontend service. +type SchedulerForFrontendServer interface { + // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and + // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. + // + // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both + // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending + // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. + FrontendLoop(SchedulerForFrontend_FrontendLoopServer) error +} + +// UnimplementedSchedulerForFrontendServer can be embedded to have forward compatible implementations. +type UnimplementedSchedulerForFrontendServer struct { +} + +func (*UnimplementedSchedulerForFrontendServer) FrontendLoop(srv SchedulerForFrontend_FrontendLoopServer) error { + return status.Errorf(codes.Unimplemented, "method FrontendLoop not implemented") +} + +func RegisterSchedulerForFrontendServer(s *grpc.Server, srv SchedulerForFrontendServer) { + s.RegisterService(&_SchedulerForFrontend_serviceDesc, srv) +} + +func _SchedulerForFrontend_FrontendLoop_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SchedulerForFrontendServer).FrontendLoop(&schedulerForFrontendFrontendLoopServer{stream}) +} + +type SchedulerForFrontend_FrontendLoopServer interface { + Send(*SchedulerToFrontend) error + Recv() (*FrontendToScheduler, error) + grpc.ServerStream +} + +type schedulerForFrontendFrontendLoopServer struct { + grpc.ServerStream +} + +func (x *schedulerForFrontendFrontendLoopServer) Send(m *SchedulerToFrontend) error { + return x.ServerStream.SendMsg(m) +} + +func (x *schedulerForFrontendFrontendLoopServer) Recv() (*FrontendToScheduler, error) { + m := new(FrontendToScheduler) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _SchedulerForFrontend_serviceDesc = grpc.ServiceDesc{ + ServiceName: "schedulerpb.SchedulerForFrontend", + HandlerType: (*SchedulerForFrontendServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "FrontendLoop", + Handler: _SchedulerForFrontend_FrontendLoop_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "scheduler.proto", +} + +func (m *QuerierToScheduler) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QuerierToScheduler) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QuerierToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.QuerierID) > 0 { + i -= len(m.QuerierID) + copy(dAtA[i:], m.QuerierID) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.QuerierID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SchedulerToQuerier) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SchedulerToQuerier) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SchedulerToQuerier) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.UserID) > 0 { + i -= len(m.UserID) + copy(dAtA[i:], m.UserID) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.UserID))) + i-- + dAtA[i] = 0x22 + } + if len(m.FrontendAddress) > 0 { + i -= len(m.FrontendAddress) + copy(dAtA[i:], m.FrontendAddress) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.FrontendAddress))) + i-- + dAtA[i] = 0x1a + } + if m.HttpRequest != nil { + { + size, err := m.HttpRequest.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintScheduler(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.QueryID != 0 { + i = encodeVarintScheduler(dAtA, i, uint64(m.QueryID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *FrontendToScheduler) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FrontendToScheduler) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.HttpRequest != nil { + { + size, err := m.HttpRequest.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintScheduler(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + if len(m.UserID) > 0 { + i -= len(m.UserID) + copy(dAtA[i:], m.UserID) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.UserID))) + i-- + dAtA[i] = 0x22 + } + if m.QueryID != 0 { + i = encodeVarintScheduler(dAtA, i, uint64(m.QueryID)) + i-- + dAtA[i] = 0x18 + } + if len(m.FrontendAddress) > 0 { + i -= len(m.FrontendAddress) + copy(dAtA[i:], m.FrontendAddress) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.FrontendAddress))) + i-- + dAtA[i] = 0x12 + } + if m.Type != 0 { + i = encodeVarintScheduler(dAtA, i, uint64(m.Type)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SchedulerToFrontend) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SchedulerToFrontend) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SchedulerToFrontend) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = encodeVarintScheduler(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0x12 + } + if m.Status != 0 { + i = encodeVarintScheduler(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintScheduler(dAtA []byte, offset int, v uint64) int { + offset -= sovScheduler(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *QuerierToScheduler) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.QuerierID) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + return n +} + +func (m *SchedulerToQuerier) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.QueryID != 0 { + n += 1 + sovScheduler(uint64(m.QueryID)) + } + if m.HttpRequest != nil { + l = m.HttpRequest.Size() + n += 1 + l + sovScheduler(uint64(l)) + } + l = len(m.FrontendAddress) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + l = len(m.UserID) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + return n +} + +func (m *FrontendToScheduler) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Type != 0 { + n += 1 + sovScheduler(uint64(m.Type)) + } + l = len(m.FrontendAddress) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + if m.QueryID != 0 { + n += 1 + sovScheduler(uint64(m.QueryID)) + } + l = len(m.UserID) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + if m.HttpRequest != nil { + l = m.HttpRequest.Size() + n += 1 + l + sovScheduler(uint64(l)) + } + return n +} + +func (m *SchedulerToFrontend) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovScheduler(uint64(m.Status)) + } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovScheduler(uint64(l)) + } + return n +} + +func sovScheduler(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozScheduler(x uint64) (n int) { + return sovScheduler(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *QuerierToScheduler) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&QuerierToScheduler{`, + `QuerierID:` + fmt.Sprintf("%v", this.QuerierID) + `,`, + `}`, + }, "") + return s +} +func (this *SchedulerToQuerier) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&SchedulerToQuerier{`, + `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, + `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, + `FrontendAddress:` + fmt.Sprintf("%v", this.FrontendAddress) + `,`, + `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, + `}`, + }, "") + return s +} +func (this *FrontendToScheduler) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&FrontendToScheduler{`, + `Type:` + fmt.Sprintf("%v", this.Type) + `,`, + `FrontendAddress:` + fmt.Sprintf("%v", this.FrontendAddress) + `,`, + `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, + `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, + `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, + `}`, + }, "") + return s +} +func (this *SchedulerToFrontend) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&SchedulerToFrontend{`, + `Status:` + fmt.Sprintf("%v", this.Status) + `,`, + `Error:` + fmt.Sprintf("%v", this.Error) + `,`, + `}`, + }, "") + return s +} +func valueToStringScheduler(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *QuerierToScheduler) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QuerierToScheduler: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QuerierToScheduler: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field QuerierID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.QuerierID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipScheduler(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SchedulerToQuerier) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SchedulerToQuerier: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SchedulerToQuerier: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) + } + m.QueryID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.QueryID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HttpRequest", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.HttpRequest == nil { + m.HttpRequest = &httpgrpc.HTTPRequest{} + } + if err := m.HttpRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FrontendAddress", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FrontendAddress = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UserID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UserID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipScheduler(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FrontendToScheduler: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FrontendToScheduler: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= FrontendToSchedulerType(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FrontendAddress", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FrontendAddress = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) + } + m.QueryID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.QueryID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UserID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UserID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HttpRequest", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.HttpRequest == nil { + m.HttpRequest = &httpgrpc.HTTPRequest{} + } + if err := m.HttpRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipScheduler(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SchedulerToFrontend) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SchedulerToFrontend: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SchedulerToFrontend: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= SchedulerToFrontendStatus(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthScheduler + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthScheduler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipScheduler(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthScheduler + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipScheduler(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowScheduler + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowScheduler + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowScheduler + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthScheduler + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthScheduler + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowScheduler + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipScheduler(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthScheduler + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthScheduler = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowScheduler = fmt.Errorf("proto: integer overflow") +) diff --git a/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.proto b/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.proto new file mode 100644 index 000000000000..62fab0d408fc --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/scheduler/schedulerpb/scheduler.proto @@ -0,0 +1,85 @@ +syntax = "proto3"; + +package schedulerpb; + +option go_package = "schedulerpb"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/weaveworks/common/httpgrpc/httpgrpc.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +// Scheduler interface exposed to Queriers. +service SchedulerForQuerier { + // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for + // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, + // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. + // + // Long-running loop is used to detect broken connection between scheduler and querier. This is important + // for scheduler to keep a list of connected queriers up-to-date. + rpc QuerierLoop(stream QuerierToScheduler) returns (stream SchedulerToQuerier) { }; +} + +// Querier reports its own clientID when it connects, so that scheduler knows how many *different* queriers are connected. +// To signal that querier is ready to accept another request, querier sends empty message. +message QuerierToScheduler { + string querierID = 1; +} + +message SchedulerToQuerier { + // Query ID as reported by frontend. When querier sends the response back to frontend (using frontendAddress), + // it identifies the query by using this ID. + uint64 queryID = 1; + httpgrpc.HTTPRequest httpRequest = 2; + + // Where should querier send HTTP Response to (using FrontendForQuerier interface). + string frontendAddress = 3; + + // User who initiated the request. Needed to send reply back to frontend. + string userID = 4; +} + +// Scheduler interface exposed to Frontend. Frontend can enqueue and cancel requests. +service SchedulerForFrontend { + // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and + // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. + // + // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both + // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending + // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. + rpc FrontendLoop(stream FrontendToScheduler) returns (stream SchedulerToFrontend) { }; +} + +enum FrontendToSchedulerType { + INIT = 0; + ENQUEUE = 1; + CANCEL = 2; +} + +message FrontendToScheduler { + FrontendToSchedulerType type = 1; + + // Used by INIT message. Will be put into all requests passed to querier. + string frontendAddress = 2; + + // Used by ENQUEUE and CANCEL. + // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. + uint64 queryID = 3; + + // Following are used by ENQUEUE only. + string userID = 4; + httpgrpc.HTTPRequest httpRequest = 5; +} + +enum SchedulerToFrontendStatus { + OK = 0; + TOO_MANY_REQUESTS_PER_TENANT = 1; + ERROR = 2; + SHUTTING_DOWN = 3; +} + +message SchedulerToFrontend { + SchedulerToFrontendStatus status = 1; + string error = 2; +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/bucket_client.go b/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/bucket_client.go new file mode 100644 index 000000000000..179647dd4fd4 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/bucket_client.go @@ -0,0 +1,37 @@ +package swift + +import ( + "github.com/go-kit/kit/log" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/swift" + yaml "gopkg.in/yaml.v2" +) + +// NewBucketClient creates a new Swift bucket client +func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { + bucketConfig := swift.SwiftConfig{ + AuthUrl: cfg.AuthURL, + Username: cfg.Username, + UserDomainName: cfg.UserDomainName, + UserDomainID: cfg.UserDomainID, + UserId: cfg.UserID, + Password: cfg.Password, + DomainId: cfg.DomainID, + DomainName: cfg.DomainName, + ProjectID: cfg.ProjectID, + ProjectName: cfg.ProjectName, + ProjectDomainID: cfg.ProjectDomainID, + ProjectDomainName: cfg.ProjectDomainName, + RegionName: cfg.RegionName, + ContainerName: cfg.ContainerName, + } + + // Thanos currently doesn't support passing the config as is, but expects a YAML, + // so we're going to serialize it. + serialized, err := yaml.Marshal(bucketConfig) + if err != nil { + return nil, err + } + + return swift.NewContainer(logger, serialized) +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/config.go b/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/config.go new file mode 100644 index 000000000000..13347fb9d98b --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/storage/backend/swift/config.go @@ -0,0 +1,46 @@ +package swift + +import ( + "flag" +) + +// Config holds the config options for Swift backend +type Config struct { + AuthURL string `yaml:"auth_url"` + Username string `yaml:"username"` + UserDomainName string `yaml:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id"` + UserID string `yaml:"user_id"` + Password string `yaml:"password"` + DomainID string `yaml:"domain_id"` + DomainName string `yaml:"domain_name"` + ProjectID string `yaml:"project_id"` + ProjectName string `yaml:"project_name"` + ProjectDomainID string `yaml:"project_domain_id"` + ProjectDomainName string `yaml:"project_domain_name"` + RegionName string `yaml:"region_name"` + ContainerName string `yaml:"container_name"` +} + +// RegisterFlags registers the flags for TSDB Swift storage +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.RegisterFlagsWithPrefix("blocks-storage.", f) +} + +// RegisterFlagsWithPrefix registers the flags for TSDB Swift storage with the provided prefix +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.AuthURL, prefix+"swift.auth-url", "", "OpenStack Swift authentication URL") + f.StringVar(&cfg.Username, prefix+"swift.username", "", "OpenStack Swift username.") + f.StringVar(&cfg.UserDomainName, prefix+"swift.user-domain-name", "", "OpenStack Swift user's domain name.") + f.StringVar(&cfg.UserDomainID, prefix+"swift.user-domain-id", "", "OpenStack Swift user's domain ID.") + f.StringVar(&cfg.UserID, prefix+"swift.user-id", "", "OpenStack Swift user ID.") + f.StringVar(&cfg.Password, prefix+"swift.password", "", "OpenStack Swift API key.") + f.StringVar(&cfg.DomainID, prefix+"swift.domain-id", "", "OpenStack Swift user's domain ID.") + f.StringVar(&cfg.DomainName, prefix+"swift.domain-name", "", "OpenStack Swift user's domain name.") + f.StringVar(&cfg.ProjectID, prefix+"swift.project-id", "", "OpenStack Swift project ID (v2,v3 auth only).") + f.StringVar(&cfg.ProjectName, prefix+"swift.project-name", "", "OpenStack Swift project name (v2,v3 auth only).") + f.StringVar(&cfg.ProjectDomainID, prefix+"swift.project-domain-id", "", "ID of the OpenStack Swift project's domain (v3 auth only), only needed if it differs the from user domain.") + f.StringVar(&cfg.ProjectDomainName, prefix+"swift.project-domain-name", "", "Name of the OpenStack Swift project's domain (v3 auth only), only needed if it differs from the user domain.") + f.StringVar(&cfg.RegionName, prefix+"swift.region-name", "", "OpenStack Swift Region to use (v2,v3 auth only).") + f.StringVar(&cfg.ContainerName, prefix+"swift.container-name", "", "Name of the OpenStack Swift container to put chunks in.") +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/bucket_client.go b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/bucket_client.go index 999fb550a587..eeb92aca6a46 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/bucket_client.go +++ b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/bucket_client.go @@ -11,6 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/storage/backend/filesystem" "github.com/cortexproject/cortex/pkg/storage/backend/gcs" "github.com/cortexproject/cortex/pkg/storage/backend/s3" + "github.com/cortexproject/cortex/pkg/storage/backend/swift" ) // NewBucketClient creates a new bucket client based on the configured backend @@ -22,6 +23,8 @@ func NewBucketClient(ctx context.Context, cfg BucketConfig, name string, logger client, err = gcs.NewBucketClient(ctx, cfg.GCS, name, logger) case BackendAzure: client, err = azure.NewBucketClient(cfg.Azure, name, logger) + case BackendSwift: + client, err = swift.NewBucketClient(cfg.Swift, name, logger) case BackendFilesystem: client, err = filesystem.NewBucketClient(cfg.Filesystem) default: diff --git a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/config.go b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/config.go index e0493f097c4c..06468c7c302c 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/config.go +++ b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/config.go @@ -9,6 +9,7 @@ import ( "github.com/alecthomas/units" "github.com/pkg/errors" + "github.com/prometheus/prometheus/tsdb/wal" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/store" @@ -16,6 +17,7 @@ import ( "github.com/cortexproject/cortex/pkg/storage/backend/filesystem" "github.com/cortexproject/cortex/pkg/storage/backend/gcs" "github.com/cortexproject/cortex/pkg/storage/backend/s3" + "github.com/cortexproject/cortex/pkg/storage/backend/swift" "github.com/cortexproject/cortex/pkg/util" ) @@ -29,6 +31,9 @@ const ( // BackendAzure is the value for the Azure storage backend BackendAzure = "azure" + // BackendSwift is the value for the Openstack Swift storage backend + BackendSwift = "swift" + // BackendFilesystem is the value for the filesystem storge backend BackendFilesystem = "filesystem" @@ -47,12 +52,14 @@ const ( // Validation errors var ( - supportedBackends = []string{BackendS3, BackendGCS, BackendAzure, BackendFilesystem} + supportedBackends = []string{BackendS3, BackendGCS, BackendAzure, BackendSwift, BackendFilesystem} errUnsupportedStorageBackend = errors.New("unsupported TSDB storage backend") errInvalidShipConcurrency = errors.New("invalid TSDB ship concurrency") + errInvalidOpeningConcurrency = errors.New("invalid TSDB opening concurrency") errInvalidCompactionInterval = errors.New("invalid TSDB compaction interval") errInvalidCompactionConcurrency = errors.New("invalid TSDB compaction concurrency") + errInvalidWALSegmentSizeBytes = errors.New("invalid TSDB WAL segment size bytes") errInvalidStripeSize = errors.New("invalid TSDB stripe size") errEmptyBlockranges = errors.New("empty block ranges for TSDB") ) @@ -64,6 +71,7 @@ type BucketConfig struct { S3 s3.Config `yaml:"s3"` GCS gcs.Config `yaml:"gcs"` Azure azure.Config `yaml:"azure"` + Swift swift.Config `yaml:"swift"` Filesystem filesystem.Config `yaml:"filesystem"` // Not used internally, meant to allow callers to wrap Buckets @@ -121,6 +129,7 @@ func (cfg *BucketConfig) RegisterFlags(f *flag.FlagSet) { cfg.S3.RegisterFlags(f) cfg.GCS.RegisterFlags(f) cfg.Azure.RegisterFlags(f) + cfg.Swift.RegisterFlags(f) cfg.Filesystem.RegisterFlags(f) f.StringVar(&cfg.Backend, "blocks-storage.backend", "s3", fmt.Sprintf("Backend storage to use. Supported backends are: %s.", strings.Join(supportedBackends, ", "))) @@ -167,6 +176,7 @@ type TSDBConfig struct { HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"` StripeSize int `yaml:"stripe_size"` WALCompressionEnabled bool `yaml:"wal_compression_enabled"` + WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes"` FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown"` // MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup. @@ -194,6 +204,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.HeadCompactionIdleTimeout, "blocks-storage.tsdb.head-compaction-idle-timeout", 1*time.Hour, "If TSDB head is idle for this duration, it is compacted. 0 means disabled.") f.IntVar(&cfg.StripeSize, "blocks-storage.tsdb.stripe-size", 16384, "The number of shards of series to use in TSDB (must be a power of 2). Reducing this will decrease memory footprint, but can negatively impact performance.") f.BoolVar(&cfg.WALCompressionEnabled, "blocks-storage.tsdb.wal-compression-enabled", false, "True to enable TSDB WAL compression.") + f.IntVar(&cfg.WALSegmentSizeBytes, "blocks-storage.tsdb.wal-segment-size-bytes", wal.DefaultSegmentSize, "TSDB WAL segments files max size (bytes).") f.BoolVar(&cfg.FlushBlocksOnShutdown, "blocks-storage.tsdb.flush-blocks-on-shutdown", false, "True to flush blocks to storage on shutdown. If false, incomplete blocks will be reused after restart.") } @@ -203,6 +214,10 @@ func (cfg *TSDBConfig) Validate() error { return errInvalidShipConcurrency } + if cfg.MaxTSDBOpeningConcurrencyOnStartup <= 0 { + return errInvalidOpeningConcurrency + } + if cfg.HeadCompactionInterval <= 0 || cfg.HeadCompactionInterval > 5*time.Minute { return errInvalidCompactionInterval } @@ -219,6 +234,10 @@ func (cfg *TSDBConfig) Validate() error { return errEmptyBlockranges } + if cfg.WALSegmentSizeBytes <= 0 { + return errInvalidWALSegmentSizeBytes + } + return nil } diff --git a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/memcache_client_config.go b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/memcache_client_config.go index 8b3fd2e307a0..60c1a4e2861d 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/memcache_client_config.go +++ b/vendor/github.com/cortexproject/cortex/pkg/storage/tsdb/memcache_client_config.go @@ -27,7 +27,7 @@ func (cfg *MemcachedClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefi f.IntVar(&cfg.MaxAsyncConcurrency, prefix+"max-async-concurrency", 50, "The maximum number of concurrent asynchronous operations can occur.") f.IntVar(&cfg.MaxAsyncBufferSize, prefix+"max-async-buffer-size", 10000, "The maximum number of enqueued asynchronous operations allowed.") f.IntVar(&cfg.MaxGetMultiConcurrency, prefix+"max-get-multi-concurrency", 100, "The maximum number of concurrent connections running get operations. If set to 0, concurrency is unlimited.") - f.IntVar(&cfg.MaxGetMultiBatchSize, prefix+"max-get-multi-batch-size", 0, "The maximum number of keys a single underlying get operation should run. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring the max concurrency. If set to 0, the max batch size is unlimited.") + f.IntVar(&cfg.MaxGetMultiBatchSize, prefix+"max-get-multi-batch-size", 0, "The maximum number of keys a single underlying get operation should run. If more keys are specified, internally keys are split into multiple batches and fetched concurrently, honoring the max concurrency. If set to 0, the max batch size is unlimited.") f.IntVar(&cfg.MaxItemSize, prefix+"max-item-size", 1024*1024, "The maximum size of an item stored in memcached. Bigger items are not stored. If set to 0, no maximum size is enforced.") } diff --git a/vendor/github.com/cortexproject/cortex/pkg/storegateway/bucket_stores.go b/vendor/github.com/cortexproject/cortex/pkg/storegateway/bucket_stores.go index a1237b77c5eb..7496b3f3dd0f 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/storegateway/bucket_stores.go +++ b/vendor/github.com/cortexproject/cortex/pkg/storegateway/bucket_stores.go @@ -154,7 +154,7 @@ func (u *BucketStores) syncUsersBlocks(ctx context.Context, f func(context.Conte wg := &sync.WaitGroup{} jobs := make(chan job) - errs := tsdb_errors.MultiError{} + errs := tsdb_errors.NewMulti() errsMx := sync.Mutex{} // Scan users in the bucket. In case of error, it may return a subset of users. If we sync a subset of users diff --git a/vendor/github.com/cortexproject/cortex/pkg/storegateway/gateway.go b/vendor/github.com/cortexproject/cortex/pkg/storegateway/gateway.go index 3ae3458131de..8fa8cf01b792 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/storegateway/gateway.go +++ b/vendor/github.com/cortexproject/cortex/pkg/storegateway/gateway.go @@ -270,7 +270,7 @@ func (g *StoreGateway) running(ctx context.Context) error { defer syncTicker.Stop() if g.gatewayCfg.ShardingEnabled { - ringLastState, _ = g.ring.GetAll(ring.BlocksSync) // nolint:errcheck + ringLastState, _ = g.ring.GetAllHealthy(ring.BlocksSync) // nolint:errcheck ringTicker := time.NewTicker(util.DurationWithJitter(g.gatewayCfg.ShardingRing.RingCheckPeriod, 0.2)) defer ringTicker.Stop() ringTickerChan = ringTicker.C @@ -283,7 +283,7 @@ func (g *StoreGateway) running(ctx context.Context) error { case <-ringTickerChan: // We ignore the error because in case of error it will return an empty // replication set which we use to compare with the previous state. - currRingState, _ := g.ring.GetAll(ring.BlocksSync) // nolint:errcheck + currRingState, _ := g.ring.GetAllHealthy(ring.BlocksSync) // nolint:errcheck if ring.HasReplicationSetChanged(ringLastState, currRingState) { ringLastState = currRingState diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/dns_watcher.go b/vendor/github.com/cortexproject/cortex/pkg/util/dns_watcher.go new file mode 100644 index 000000000000..d4af88f57ba6 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/util/dns_watcher.go @@ -0,0 +1,82 @@ +package util + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "google.golang.org/grpc/naming" + + "github.com/cortexproject/cortex/pkg/util/services" +) + +// Notifications about address resolution. All notifications are sent on the same goroutine. +type DNSNotifications interface { + // New address has been discovered by DNS watcher for supplied hostname. + AddressAdded(address string) + + // Previously-discovered address is no longer resolved for the hostname. + AddressRemoved(address string) +} + +type dnsWatcher struct { + watcher naming.Watcher //nolint:staticcheck //Skipping for now. If you still see this more than likely issue https://github.com/cortexproject/cortex/issues/2015 has not yet been addressed. + notifications DNSNotifications +} + +// NewDNSWatcher creates a new DNS watcher and returns a service that is wrapping it. +func NewDNSWatcher(address string, dnsLookupPeriod time.Duration, notifications DNSNotifications) (services.Service, error) { + resolver, err := naming.NewDNSResolverWithFreq(dnsLookupPeriod) + if err != nil { + return nil, err + } + + watcher, err := resolver.Resolve(address) + if err != nil { + return nil, err + } + + w := &dnsWatcher{ + watcher: watcher, + notifications: notifications, + } + return services.NewBasicService(nil, w.watchDNSLoop, nil), nil +} + +// watchDNSLoop watches for changes in DNS and sends notifications. +func (w *dnsWatcher) watchDNSLoop(servCtx context.Context) error { + go func() { + // Close the watcher, when this service is asked to stop. + // Closing the watcher makes watchDNSLoop exit, since it only iterates on watcher updates, and has no other + // way to stop. We cannot close the watcher in `stopping` method, because it is only called *after* + // watchDNSLoop exits. + <-servCtx.Done() + w.watcher.Close() + }() + + for { + updates, err := w.watcher.Next() + if err != nil { + // watcher.Next returns error when Close is called, but we call Close when our context is done. + // we don't want to report error in that case. + if servCtx.Err() != nil { + return nil + } + return errors.Wrapf(err, "error from DNS watcher") + } + + for _, update := range updates { + switch update.Op { + case naming.Add: + w.notifications.AddressAdded(update.Addr) + + case naming.Delete: + w.notifications.AddressRemoved(update.Addr) + + default: + return fmt.Errorf("unknown op: %v", update.Op) + } + } + } +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/errors.go b/vendor/github.com/cortexproject/cortex/pkg/util/errors.go index c372e819466b..0299ad25f7af 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/errors.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/errors.go @@ -1,6 +1,8 @@ package util -import "errors" +import ( + "errors" +) // ErrStopProcess is the error returned by a service as a hint to stop the server entirely. var ErrStopProcess = errors.New("stop process") diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/fakeauth/fake_auth.go b/vendor/github.com/cortexproject/cortex/pkg/util/fakeauth/fake_auth.go index 42557e15240d..ee850e804516 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/fakeauth/fake_auth.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/fakeauth/fake_auth.go @@ -15,19 +15,27 @@ import ( // SetupAuthMiddleware for the given server config. func SetupAuthMiddleware(config *server.Config, enabled bool, noGRPCAuthOn []string) middleware.Interface { if enabled { - config.GRPCMiddleware = append(config.GRPCMiddleware, - middleware.ServerUserHeaderInterceptor, - ) + ignoredMethods := map[string]bool{} + for _, m := range noGRPCAuthOn { + ignoredMethods[m] = true + } + + config.GRPCMiddleware = append(config.GRPCMiddleware, func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + if ignoredMethods[info.FullMethod] { + return handler(ctx, req) + } + return middleware.ServerUserHeaderInterceptor(ctx, req, info, handler) + }) + config.GRPCStreamMiddleware = append(config.GRPCStreamMiddleware, func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - for _, path := range noGRPCAuthOn { - if info.FullMethod == path { - return handler(srv, ss) - } + if ignoredMethods[info.FullMethod] { + return handler(srv, ss) } return middleware.StreamServerUserHeaderInterceptor(srv, ss, info, handler) }, ) + return middleware.AuthenticateUser } diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/grpcclient/grpcclient.go b/vendor/github.com/cortexproject/cortex/pkg/util/grpcclient/grpcclient.go index 8a73616946b5..522011fb01c6 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/grpcclient/grpcclient.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/grpcclient/grpcclient.go @@ -2,6 +2,7 @@ package grpcclient import ( "flag" + "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -9,6 +10,7 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/keepalive" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" @@ -90,6 +92,11 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep grpc.WithDefaultCallOptions(cfg.CallOptions()...), grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)), grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: time.Second * 20, + Timeout: time.Second * 10, + PermitWithoutStream: true, + }), } } diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/grpcutil/carrier.go b/vendor/github.com/cortexproject/cortex/pkg/util/grpcutil/carrier.go new file mode 100644 index 000000000000..1c6dee7ff5a2 --- /dev/null +++ b/vendor/github.com/cortexproject/cortex/pkg/util/grpcutil/carrier.go @@ -0,0 +1,40 @@ +package grpcutil + +import ( + "github.com/opentracing/opentracing-go" + "github.com/weaveworks/common/httpgrpc" +) + +// Used to transfer trace information from/to HTTP request. +type HttpgrpcHeadersCarrier httpgrpc.HTTPRequest + +func (c *HttpgrpcHeadersCarrier) Set(key, val string) { + c.Headers = append(c.Headers, &httpgrpc.Header{ + Key: key, + Values: []string{val}, + }) +} + +func (c *HttpgrpcHeadersCarrier) ForeachKey(handler func(key, val string) error) error { + for _, h := range c.Headers { + for _, v := range h.Values { + if err := handler(h.Key, v); err != nil { + return err + } + } + } + return nil +} + +func GetParentSpanForRequest(tracer opentracing.Tracer, req *httpgrpc.HTTPRequest) (opentracing.SpanContext, error) { + if tracer == nil { + return nil, nil + } + + carrier := (*HttpgrpcHeadersCarrier)(req) + extracted, err := tracer.Extract(opentracing.HTTPHeaders, carrier) + if err == opentracing.ErrSpanContextNotFound { + err = nil + } + return extracted, err +} diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/metrics_helper.go b/vendor/github.com/cortexproject/cortex/pkg/util/metrics_helper.go index 20c611967530..86ead8f79cc9 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/metrics_helper.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/metrics_helper.go @@ -228,6 +228,13 @@ func (d MetricFamiliesPerUser) SendMaxOfGauges(out chan<- prometheus.Metric, des out <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, result) } +func (d MetricFamiliesPerUser) SendMaxOfGaugesPerUser(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) { + for user, userMetrics := range d { + result := userMetrics.MaxGauges(gauge) + out <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, result, user) + } +} + func (d MetricFamiliesPerUser) SendSumOfSummaries(out chan<- prometheus.Metric, desc *prometheus.Desc, summaryName string) { summaryData := SummaryData{} for _, userMetrics := range d { diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/spanlogger/spanlogger.go b/vendor/github.com/cortexproject/cortex/pkg/util/spanlogger/spanlogger.go index b376d2585e5e..87a11baafc8f 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/spanlogger/spanlogger.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/spanlogger/spanlogger.go @@ -32,7 +32,7 @@ func New(ctx context.Context, method string, kvps ...interface{}) (*SpanLogger, } // FromContext returns a span logger using the current parent span. -// If there is no parent span, the Spanlogger will only log to stdout. +// If there is no parent span, the Spanlogger will only log to global logger. func FromContext(ctx context.Context) *SpanLogger { sp := opentracing.SpanFromContext(ctx) if sp == nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/test/poll.go b/vendor/github.com/cortexproject/cortex/pkg/util/test/poll.go index 3429a8525865..fdd33264c947 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/test/poll.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/test/poll.go @@ -21,6 +21,6 @@ func Poll(t testing.TB, d time.Duration, want interface{}, have func() interface } h := have() if !reflect.DeepEqual(want, h) { - t.Fatalf("%v != %v", want, h) + t.Fatalf("expected %v, got %v", want, h) } } diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/time.go b/vendor/github.com/cortexproject/cortex/pkg/util/time.go index 45a4624565df..7b55613e61ce 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/time.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/time.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/prometheus/common/model" "github.com/weaveworks/common/httpgrpc" ) @@ -23,6 +24,16 @@ func TimeFromMillis(ms int64) time.Time { return time.Unix(0, ms*nanosecondsInMillisecond) } +// FormatTimeMillis returns a human readable version of the input time (in milliseconds). +func FormatTimeMillis(ms int64) string { + return TimeFromMillis(ms).String() +} + +// FormatTimeModel returns a human readable version of the input time. +func FormatTimeModel(t model.Time) string { + return TimeFromMillis(int64(t)).String() +} + // ParseTime parses the string into an int64, milliseconds since epoch. func ParseTime(s string) (int64, error) { if t, err := strconv.ParseFloat(s, 64); err == nil { diff --git a/vendor/github.com/cortexproject/cortex/pkg/util/validation/limits.go b/vendor/github.com/cortexproject/cortex/pkg/util/validation/limits.go index eb4e0ddafdbb..1a99f9d8f16d 100644 --- a/vendor/github.com/cortexproject/cortex/pkg/util/validation/limits.go +++ b/vendor/github.com/cortexproject/cortex/pkg/util/validation/limits.go @@ -5,6 +5,8 @@ import ( "flag" "time" + "github.com/prometheus/prometheus/pkg/relabel" + "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -46,6 +48,7 @@ type Limits struct { EnforceMetadataMetricName bool `yaml:"enforce_metadata_metric_name"` EnforceMetricName bool `yaml:"enforce_metric_name"` IngestionTenantShardSize int `yaml:"ingestion_tenant_shard_size"` + MetricRelabelConfigs []*relabel.Config `yaml:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs."` // Ingester enforced limits. // Series @@ -64,6 +67,7 @@ type Limits struct { // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_chunks_per_query"` + MaxQueryLookback time.Duration `yaml:"max_query_lookback"` MaxQueryLength time.Duration `yaml:"max_query_length"` MaxQueryParallelism int `yaml:"max_query_parallelism"` CardinalityLimit int `yaml:"cardinality_limit"` @@ -119,10 +123,11 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxChunksPerQuery, "store.query-chunk-limit", 2e6, "Maximum number of chunks that can be fetched in a single query. This limit is enforced when fetching chunks from the long-term storage. When running the Cortex chunks storage, this limit is enforced in the querier, while when running the Cortex blocks storage this limit is both enforced in the querier and store-gateway. 0 to disable.") f.DurationVar(&l.MaxQueryLength, "store.max-query-length", 0, "Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and in the chunks storage. 0 to disable.") - f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of queries will be scheduled in parallel by the frontend.") + f.DurationVar(&l.MaxQueryLookback, "querier.max-query-lookback", 0, "Limit how long back data (series and metadata) can be queried, up until duration ago. This limit is enforced in the query-frontend, querier and ruler. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") + f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of split queries will be scheduled in parallel by the frontend.") f.IntVar(&l.CardinalityLimit, "store.cardinality-limit", 1e5, "Cardinality limit for index queries. This limit is ignored when running the Cortex blocks storage. 0 to disable.") f.DurationVar(&l.MaxCacheFreshness, "frontend.max-cache-freshness", 1*time.Minute, "Most recent allowed cacheable result per-tenant, to prevent caching very recent results that might still be in flux.") - f.IntVar(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends). This option only works with queriers connecting to the query-frontend, not when using downstream URL.") + f.IntVar(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.") f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") @@ -305,12 +310,18 @@ func (o *Overrides) MaxChunksPerQuery(userID string) int { return o.getOverridesForUser(userID).MaxChunksPerQuery } +// MaxQueryLookback returns the max lookback period of queries. +func (o *Overrides) MaxQueryLookback(userID string) time.Duration { + return o.getOverridesForUser(userID).MaxQueryLookback +} + // MaxQueryLength returns the limit of the length (in time) of a query. func (o *Overrides) MaxQueryLength(userID string) time.Duration { return o.getOverridesForUser(userID).MaxQueryLength } -// MaxCacheFreshness returns the limit of the length (in time) of a query. +// MaxCacheFreshness returns the period after which results are cacheable, +// to prevent caching of very recent results. func (o *Overrides) MaxCacheFreshness(userID string) time.Duration { return o.getOverridesForUser(userID).MaxCacheFreshness } @@ -320,7 +331,7 @@ func (o *Overrides) MaxQueriersPerUser(userID string) int { return o.getOverridesForUser(userID).MaxQueriersPerTenant } -// MaxQueryParallelism returns the limit to the number of sub-queries the +// MaxQueryParallelism returns the limit to the number of split queries the // frontend will process in parallel. func (o *Overrides) MaxQueryParallelism(userID string) int { return o.getOverridesForUser(userID).MaxQueryParallelism @@ -376,6 +387,11 @@ func (o *Overrides) EvaluationDelay(userID string) time.Duration { return o.getOverridesForUser(userID).RulerEvaluationDelay } +// MetricRelabelConfigs returns the metric relabel configs for a given user. +func (o *Overrides) MetricRelabelConfigs(userID string) []*relabel.Config { + return o.getOverridesForUser(userID).MetricRelabelConfigs +} + // RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy. func (o *Overrides) RulerTenantShardSize(userID string) int { return o.getOverridesForUser(userID).RulerTenantShardSize diff --git a/vendor/github.com/prometheus/prometheus/discovery/discovery.go b/vendor/github.com/prometheus/prometheus/discovery/discovery.go index d77b193b3585..5b0402bdb026 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/discovery.go +++ b/vendor/github.com/prometheus/prometheus/discovery/discovery.go @@ -18,8 +18,8 @@ import ( "reflect" "github.com/go-kit/kit/log" - "github.com/prometheus/common/config" + "github.com/prometheus/prometheus/discovery/targetgroup" ) diff --git a/vendor/github.com/prometheus/prometheus/discovery/dockerswarm/network.go b/vendor/github.com/prometheus/prometheus/discovery/dockerswarm/network.go index 92e5ad888249..7d70169a67dd 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/dockerswarm/network.go +++ b/vendor/github.com/prometheus/prometheus/discovery/dockerswarm/network.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/docker/docker/api/types" + "github.com/prometheus/prometheus/util/strutil" ) diff --git a/vendor/github.com/prometheus/prometheus/discovery/openstack/instance.go b/vendor/github.com/prometheus/prometheus/discovery/openstack/instance.go index 8ae4a05b70dd..a53d6274ef70 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/openstack/instance.go +++ b/vendor/github.com/prometheus/prometheus/discovery/openstack/instance.go @@ -27,6 +27,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" ) diff --git a/vendor/github.com/prometheus/prometheus/discovery/registry.go b/vendor/github.com/prometheus/prometheus/discovery/registry.go index 5f9a40546ae9..2ebb36cb2907 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/registry.go +++ b/vendor/github.com/prometheus/prometheus/discovery/registry.go @@ -21,8 +21,9 @@ import ( "strings" "sync" - "github.com/prometheus/prometheus/discovery/targetgroup" "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery/targetgroup" ) const ( diff --git a/vendor/github.com/prometheus/prometheus/notifier/notifier.go b/vendor/github.com/prometheus/prometheus/notifier/notifier.go index 29545bbee0ee..5856613503ef 100644 --- a/vendor/github.com/prometheus/prometheus/notifier/notifier.go +++ b/vendor/github.com/prometheus/prometheus/notifier/notifier.go @@ -32,13 +32,13 @@ import ( "github.com/go-kit/kit/log/level" "github.com/go-openapi/strfmt" "github.com/pkg/errors" - "go.uber.org/atomic" - "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" + "go.uber.org/atomic" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" diff --git a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go index b6b859fa7ff1..5c11cc2eeef0 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go +++ b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go @@ -19,7 +19,7 @@ import ( "sort" "strconv" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" ) // Well-known label names used by Prometheus components. @@ -29,10 +29,11 @@ const ( BucketLabel = "le" InstanceName = "instance" - sep = '\xff' labelSep = '\xfe' ) +var seps = []byte{'\xff'} + // Label is a key/value pair of strings. type Label struct { Name, Value string @@ -70,10 +71,10 @@ func (ls Labels) Bytes(buf []byte) []byte { b.WriteByte(labelSep) for i, l := range ls { if i > 0 { - b.WriteByte(sep) + b.WriteByte(seps[0]) } b.WriteString(l.Name) - b.WriteByte(sep) + b.WriteByte(seps[0]) b.WriteString(l.Value) } return b.Bytes() @@ -134,13 +135,26 @@ func (ls Labels) MatchLabels(on bool, names ...string) Labels { // Hash returns a hash value for the label set. func (ls Labels) Hash() uint64 { + // Use xxhash.Sum64(b) for fast path as it's faster. b := make([]byte, 0, 1024) + for i, v := range ls { + if len(b)+len(v.Name)+len(v.Value)+2 >= cap(b) { + // If labels entry is 1KB+ do not allocate whole entry. + h := xxhash.New() + _, _ = h.Write(b) + for _, v := range ls[i:] { + _, _ = h.WriteString(v.Name) + _, _ = h.Write(seps) + _, _ = h.WriteString(v.Value) + _, _ = h.Write(seps) + } + return h.Sum64() + } - for _, v := range ls { b = append(b, v.Name...) - b = append(b, sep) + b = append(b, seps[0]) b = append(b, v.Value...) - b = append(b, sep) + b = append(b, seps[0]) } return xxhash.Sum64(b) } @@ -157,9 +171,9 @@ func (ls Labels) HashForLabels(b []byte, names ...string) (uint64, []byte) { i++ } else { b = append(b, ls[i].Name...) - b = append(b, sep) + b = append(b, seps[0]) b = append(b, ls[i].Value...) - b = append(b, sep) + b = append(b, seps[0]) i++ j++ } @@ -181,9 +195,9 @@ func (ls Labels) HashWithoutLabels(b []byte, names ...string) (uint64, []byte) { continue } b = append(b, ls[i].Name...) - b = append(b, sep) + b = append(b, seps[0]) b = append(b, ls[i].Value...) - b = append(b, sep) + b = append(b, seps[0]) } return xxhash.Sum64(b), b } diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go index cfcd05e210f0..557e566622ba 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go @@ -85,12 +85,12 @@ const ( type MetricType string const ( - MetricTypeCounter = "counter" - MetricTypeGauge = "gauge" - MetricTypeHistogram = "histogram" - MetricTypeGaugeHistogram = "gaugehistogram" - MetricTypeSummary = "summary" - MetricTypeInfo = "info" - MetricTypeStateset = "stateset" - MetricTypeUnknown = "unknown" + MetricTypeCounter = MetricType("counter") + MetricTypeGauge = MetricType("gauge") + MetricTypeHistogram = MetricType("histogram") + MetricTypeGaugeHistogram = MetricType("gaugehistogram") + MetricTypeSummary = MetricType("summary") + MetricTypeInfo = MetricType("info") + MetricTypeStateset = MetricType("stateset") + MetricTypeUnknown = MetricType("unknown") ) diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go index cf26fcc47249..4117f0e71507 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go @@ -3,11 +3,8 @@ //line generated_parser.y:15 package parser -import __yyfmt__ "fmt" - -//line generated_parser.y:15 - import ( + __yyfmt__ "fmt" "math" "sort" "strconv" @@ -15,7 +12,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" -) +) //line generated_parser.y:15 //line generated_parser.y:28 type yySymType struct { diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/printer.go b/vendor/github.com/prometheus/prometheus/promql/parser/printer.go index e187000af743..eef4aa8e418a 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/printer.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/printer.go @@ -20,6 +20,7 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" ) diff --git a/vendor/github.com/prometheus/prometheus/promql/test.go b/vendor/github.com/prometheus/prometheus/promql/test.go index bcd90200f876..7992c5b23a88 100644 --- a/vendor/github.com/prometheus/prometheus/promql/test.go +++ b/vendor/github.com/prometheus/prometheus/promql/test.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" diff --git a/vendor/github.com/prometheus/prometheus/promql/value.go b/vendor/github.com/prometheus/prometheus/promql/value.go index a28f06bbae86..fa3a71d352d3 100644 --- a/vendor/github.com/prometheus/prometheus/promql/value.go +++ b/vendor/github.com/prometheus/prometheus/promql/value.go @@ -21,11 +21,10 @@ import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" ) func (Matrix) Type() parser.ValueType { return parser.ValueTypeMatrix } diff --git a/vendor/github.com/prometheus/prometheus/rules/alerting.go b/vendor/github.com/prometheus/prometheus/rules/alerting.go index c82bc9640acd..a4074221f9fb 100644 --- a/vendor/github.com/prometheus/prometheus/rules/alerting.go +++ b/vendor/github.com/prometheus/prometheus/rules/alerting.go @@ -16,19 +16,17 @@ package rules import ( "context" "fmt" + html_template "html/template" "net/url" "strings" "sync" "time" - html_template "html/template" - - yaml "gopkg.in/yaml.v2" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" + yaml "gopkg.in/yaml.v2" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" diff --git a/vendor/github.com/prometheus/prometheus/scrape/manager.go b/vendor/github.com/prometheus/prometheus/scrape/manager.go index 26bb0d1a5754..bf73561a1251 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/manager.go +++ b/vendor/github.com/prometheus/prometheus/scrape/manager.go @@ -26,8 +26,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index d52b0ac7f314..f6aa49810db7 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -192,7 +192,13 @@ type scrapePool struct { appendable storage.Appendable logger log.Logger - mtx sync.Mutex + // targetMtx protects activeTargets and droppedTargets from concurrent reads + // and writes. Only one of Sync/stop/reload may be called at once due to + // manager.mtxScrape so we only need to protect from concurrent reads from + // the ActiveTargets and DroppedTargets methods. This allows those two + // methods to always complete without having to wait on scrape loops to gracefull stop. + targetMtx sync.Mutex + config *config.ScrapeConfig client *http.Client // Targets and loops must always be synchronized to have the same @@ -273,8 +279,8 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, jitterSeed } func (sp *scrapePool) ActiveTargets() []*Target { - sp.mtx.Lock() - defer sp.mtx.Unlock() + sp.targetMtx.Lock() + defer sp.targetMtx.Unlock() var tActive []*Target for _, t := range sp.activeTargets { @@ -284,8 +290,8 @@ func (sp *scrapePool) ActiveTargets() []*Target { } func (sp *scrapePool) DroppedTargets() []*Target { - sp.mtx.Lock() - defer sp.mtx.Unlock() + sp.targetMtx.Lock() + defer sp.targetMtx.Unlock() return sp.droppedTargets } @@ -294,8 +300,7 @@ func (sp *scrapePool) stop() { sp.cancel() var wg sync.WaitGroup - sp.mtx.Lock() - defer sp.mtx.Unlock() + sp.targetMtx.Lock() for fp, l := range sp.loops { wg.Add(1) @@ -308,6 +313,9 @@ func (sp *scrapePool) stop() { delete(sp.loops, fp) delete(sp.activeTargets, fp) } + + sp.targetMtx.Unlock() + wg.Wait() sp.client.CloseIdleConnections() @@ -326,9 +334,6 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { targetScrapePoolReloads.Inc() start := time.Now() - sp.mtx.Lock() - defer sp.mtx.Unlock() - client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, false, false) if err != nil { targetScrapePoolReloadsFailed.Inc() @@ -352,6 +357,8 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { mrc = sp.config.MetricRelabelConfigs ) + sp.targetMtx.Lock() + forcedErr := sp.refreshTargetLimitErr() for fp, oldLoop := range sp.loops { var cache *scrapeCache @@ -387,6 +394,8 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { sp.loops[fp] = newLoop } + sp.targetMtx.Unlock() + wg.Wait() oldClient.CloseIdleConnections() targetReloadIntervalLength.WithLabelValues(interval.String()).Observe( @@ -398,11 +407,9 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { // Sync converts target groups into actual scrape targets and synchronizes // the currently running scraper with the resulting set and returns all scraped and dropped targets. func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { - sp.mtx.Lock() - defer sp.mtx.Unlock() - start := time.Now() + sp.targetMtx.Lock() var all []*Target sp.droppedTargets = []*Target{} for _, tg := range tgs { @@ -419,6 +426,7 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { } } } + sp.targetMtx.Unlock() sp.sync(all) targetSyncIntervalLength.WithLabelValues(sp.config.JobName).Observe( @@ -431,7 +439,6 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { // scrape loops for new targets, and stops scrape loops for disappeared targets. // It returns after all stopped scrape loops terminated. func (sp *scrapePool) sync(targets []*Target) { - // This function expects that you have acquired the sp.mtx lock. var ( uniqueLoops = make(map[uint64]loop) interval = time.Duration(sp.config.ScrapeInterval) @@ -442,6 +449,7 @@ func (sp *scrapePool) sync(targets []*Target) { mrc = sp.config.MetricRelabelConfigs ) + sp.targetMtx.Lock() for _, t := range targets { hash := t.hash() @@ -487,6 +495,8 @@ func (sp *scrapePool) sync(targets []*Target) { } } + sp.targetMtx.Unlock() + targetScrapePoolTargetsAdded.WithLabelValues(sp.config.JobName).Set(float64(len(uniqueLoops))) forcedErr := sp.refreshTargetLimitErr() for _, l := range sp.loops { @@ -507,7 +517,6 @@ func (sp *scrapePool) sync(targets []*Target) { // refreshTargetLimitErr returns an error that can be passed to the scrape loops // if the number of targets exceeds the configured limit. func (sp *scrapePool) refreshTargetLimitErr() error { - // This function expects that you have acquired the sp.mtx lock. if sp.config == nil || sp.config.TargetLimit == 0 && !sp.targetLimitHit { return nil } diff --git a/vendor/github.com/prometheus/prometheus/storage/fanout.go b/vendor/github.com/prometheus/prometheus/storage/fanout.go index 62b2ef54ae0a..4bc3db12d9f6 100644 --- a/vendor/github.com/prometheus/prometheus/storage/fanout.go +++ b/vendor/github.com/prometheus/prometheus/storage/fanout.go @@ -19,6 +19,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) @@ -79,8 +80,7 @@ func (f *fanout) Querier(ctx context.Context, mint, maxt int64) (Querier, error) querier, err := storage.Querier(ctx, mint, maxt) if err != nil { // Close already open Queriers, append potential errors to returned error. - errs := tsdb_errors.MultiError{err} - errs.Add(primary.Close()) + errs := tsdb_errors.NewMulti(err, primary.Close()) for _, q := range secondaries { errs.Add(q.Close()) } @@ -102,8 +102,7 @@ func (f *fanout) ChunkQuerier(ctx context.Context, mint, maxt int64) (ChunkQueri querier, err := storage.ChunkQuerier(ctx, mint, maxt) if err != nil { // Close already open Queriers, append potential errors to returned error. - errs := tsdb_errors.MultiError{err} - errs.Add(primary.Close()) + errs := tsdb_errors.NewMulti(err, primary.Close()) for _, q := range secondaries { errs.Add(q.Close()) } @@ -129,8 +128,7 @@ func (f *fanout) Appender(ctx context.Context) Appender { // Close closes the storage and all its underlying resources. func (f *fanout) Close() error { - errs := tsdb_errors.MultiError{} - errs.Add(f.primary.Close()) + errs := tsdb_errors.NewMulti(f.primary.Close()) for _, s := range f.secondaries { errs.Add(s.Close()) } diff --git a/vendor/github.com/prometheus/prometheus/storage/merge.go b/vendor/github.com/prometheus/prometheus/storage/merge.go index e3026cf5eed3..27e701883ece 100644 --- a/vendor/github.com/prometheus/prometheus/storage/merge.go +++ b/vendor/github.com/prometheus/prometheus/storage/merge.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -247,7 +248,7 @@ func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) { // Close releases the resources of the generic querier. func (q *mergeGenericQuerier) Close() error { - errs := tsdb_errors.MultiError{} + errs := tsdb_errors.NewMulti() for _, querier := range q.queriers { if err := querier.Close(); err != nil { errs.Add(err) @@ -533,11 +534,9 @@ func (c *chainSampleIterator) Next() bool { } func (c *chainSampleIterator) Err() error { - var errs tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for _, iter := range c.iterators { - if err := iter.Err(); err != nil { - errs.Add(err) - } + errs.Add(iter.Err()) } return errs.Err() } @@ -680,11 +679,9 @@ func (c *compactChunkIterator) Next() bool { } func (c *compactChunkIterator) Err() error { - var errs tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for _, iter := range c.iterators { - if err := iter.Err(); err != nil { - errs.Add(err) - } + errs.Add(iter.Err()) } errs.Add(c.err) return errs.Err() diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/client.go b/vendor/github.com/prometheus/prometheus/storage/remote/client.go index b7965a2c98c2..4e2b9e5b8ff6 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/client.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/client.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" + "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -34,7 +35,6 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/version" - "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/prometheus/prometheus/prompb" ) diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go index 47360dd57ba6..474e4e40074a 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go @@ -24,6 +24,7 @@ import ( "github.com/golang/snappy" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/intern.go b/vendor/github.com/prometheus/prometheus/storage/remote/intern.go index 98eec34141a6..23047acd9bb0 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/intern.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/intern.go @@ -21,10 +21,9 @@ package remote import ( "sync" - "go.uber.org/atomic" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "go.uber.org/atomic" ) var noReferenceReleases = promauto.NewCounter(prometheus.CounterOpts{ diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/max_gauge.go b/vendor/github.com/prometheus/prometheus/storage/remote/max_timestamp.go similarity index 80% rename from vendor/github.com/prometheus/prometheus/storage/remote/max_gauge.go rename to vendor/github.com/prometheus/prometheus/storage/remote/max_timestamp.go index a56c2047d560..3a0a6d6fd4b7 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/max_gauge.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/max_timestamp.go @@ -19,13 +19,13 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -type maxGauge struct { +type maxTimestamp struct { mtx sync.Mutex value float64 prometheus.Gauge } -func (m *maxGauge) Set(value float64) { +func (m *maxTimestamp) Set(value float64) { m.mtx.Lock() defer m.mtx.Unlock() if value > m.value { @@ -34,8 +34,14 @@ func (m *maxGauge) Set(value float64) { } } -func (m *maxGauge) Get() float64 { +func (m *maxTimestamp) Get() float64 { m.mtx.Lock() defer m.mtx.Unlock() return m.value } + +func (m *maxTimestamp) Collect(c chan<- prometheus.Metric) { + if m.Get() > 0 { + m.Gauge.Collect(c) + } +} diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go index 1a79d6c0ce96..3bc19b7b6496 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go @@ -26,9 +26,9 @@ import ( "github.com/golang/snappy" "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" @@ -56,7 +56,7 @@ type queueManagerMetrics struct { droppedSamplesTotal prometheus.Counter enqueueRetriesTotal prometheus.Counter sentBatchDuration prometheus.Histogram - highestSentTimestamp *maxGauge + highestSentTimestamp *maxTimestamp pendingSamples prometheus.Gauge shardCapacity prometheus.Gauge numShards prometheus.Gauge @@ -64,6 +64,7 @@ type queueManagerMetrics struct { minNumShards prometheus.Gauge desiredNumShards prometheus.Gauge bytesSent prometheus.Counter + maxSamplesPerSend prometheus.Gauge } func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManagerMetrics { @@ -118,7 +119,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), ConstLabels: constLabels, }) - m.highestSentTimestamp = &maxGauge{ + m.highestSentTimestamp = &maxTimestamp{ Gauge: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, @@ -176,6 +177,13 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager Help: "The total number of bytes sent by the queue.", ConstLabels: constLabels, }) + m.maxSamplesPerSend = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "max_samples_per_send", + Help: "The maximum number of samples to be sent, in a single request, to the remote storage.", + ConstLabels: constLabels, + }) return m } @@ -197,6 +205,7 @@ func (m *queueManagerMetrics) register() { m.minNumShards, m.desiredNumShards, m.bytesSent, + m.maxSamplesPerSend, ) } } @@ -217,6 +226,7 @@ func (m *queueManagerMetrics) unregister() { m.reg.Unregister(m.minNumShards) m.reg.Unregister(m.desiredNumShards) m.reg.Unregister(m.bytesSent) + m.reg.Unregister(m.maxSamplesPerSend) } } @@ -262,7 +272,7 @@ type QueueManager struct { metrics *queueManagerMetrics interner *pool - highestRecvTimestamp *maxGauge + highestRecvTimestamp *maxTimestamp } // NewQueueManager builds a new QueueManager. @@ -279,7 +289,7 @@ func NewQueueManager( client WriteClient, flushDeadline time.Duration, interner *pool, - highestRecvTimestamp *maxGauge, + highestRecvTimestamp *maxTimestamp, ) *QueueManager { if logger == nil { logger = log.NewNopLogger() @@ -372,6 +382,7 @@ func (t *QueueManager) Start() { t.metrics.maxNumShards.Set(float64(t.cfg.MaxShards)) t.metrics.minNumShards.Set(float64(t.cfg.MinShards)) t.metrics.desiredNumShards.Set(float64(t.cfg.MinShards)) + t.metrics.maxSamplesPerSend.Set(float64(t.cfg.MaxSamplesPerSend)) t.shards.start(t.numShards) t.watcher.Start() diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/read.go b/vendor/github.com/prometheus/prometheus/storage/remote/read.go index 39822a6763f7..4718b479786d 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/read.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/read.go @@ -17,6 +17,7 @@ import ( "context" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" ) diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go index 36d7c011456e..5fb4bfbecbd3 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go @@ -22,10 +22,10 @@ import ( "time" "github.com/go-kit/kit/log" - "gopkg.in/yaml.v2" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/logging" diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/write.go b/vendor/github.com/prometheus/prometheus/storage/remote/write.go index 8b0bf7622f3a..64d3be326dca 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/write.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/write.go @@ -22,6 +22,7 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" @@ -53,7 +54,7 @@ type WriteStorage struct { interner *pool // For timestampTracker. - highestTimestamp *maxGauge + highestTimestamp *maxTimestamp } // NewWriteStorage creates and runs a WriteStorage. @@ -71,7 +72,7 @@ func NewWriteStorage(logger log.Logger, reg prometheus.Registerer, walDir string samplesIn: newEWMARate(ewmaWeight, shardUpdateDuration), walDir: walDir, interner: newPool(), - highestTimestamp: &maxGauge{ + highestTimestamp: &maxTimestamp{ Gauge: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, @@ -202,7 +203,7 @@ type timestampTracker struct { writeStorage *WriteStorage samples int64 highestTimestamp int64 - highestRecvTimestamp *maxGauge + highestRecvTimestamp *maxTimestamp } // Add implements storage.Appender. diff --git a/vendor/github.com/prometheus/prometheus/template/template.go b/vendor/github.com/prometheus/prometheus/template/template.go index 0a31c4e54a4a..10a9241c7964 100644 --- a/vendor/github.com/prometheus/prometheus/template/template.go +++ b/vendor/github.com/prometheus/prometheus/template/template.go @@ -17,20 +17,19 @@ import ( "bytes" "context" "fmt" + html_template "html/template" "math" "net/url" "regexp" "sort" "strings" - "time" - - html_template "html/template" text_template "text/template" + "time" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/CHANGELOG.md b/vendor/github.com/prometheus/prometheus/tsdb/CHANGELOG.md index 844ab5b37c69..66d07bf3cc7a 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/CHANGELOG.md +++ b/vendor/github.com/prometheus/prometheus/tsdb/CHANGELOG.md @@ -33,7 +33,7 @@ - [BUGFIX] Don't panic and recover nicely when running out of disk space. - [BUGFIX] Correctly handle empty labels. - [BUGFIX] Don't crash on an unknown tombstone ref. - - [ENHANCEMENT] Re-add FromData function to create a chunk from bytes. It is used by Cortex and Thanos. + - [ENHANCEMENT] Re-add `FromData` function to create a chunk from bytes. It is used by Cortex and Thanos. - [ENHANCEMENT] Simplify mergedPostings.Seek. - [FEATURE] Added `currentSegment` metric for the current WAL segment it is being written to. diff --git a/vendor/github.com/prometheus/prometheus/tsdb/README.md b/vendor/github.com/prometheus/prometheus/tsdb/README.md index 61f867088203..248004b9d39e 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/README.md +++ b/vendor/github.com/prometheus/prometheus/tsdb/README.md @@ -2,7 +2,7 @@ [![GoDoc](https://godoc.org/github.com/prometheus/prometheus/tsdb?status.svg)](https://godoc.org/github.com/prometheus/prometheus/tsdb) -This repository contains the Prometheus storage layer that is used in its 2.x releases. +This directory contains the Prometheus storage layer that is used in its 2.x releases. A writeup of its design can be found [here](https://fabxc.org/blog/2017-04-10-writing-a-tsdb/). @@ -11,3 +11,9 @@ Based on the Gorilla TSDB [white papers](http://www.vldb.org/pvldb/vol8/p1816-te Video: [Storing 16 Bytes at Scale](https://youtu.be/b_pEevMAC3I) from [PromCon 2017](https://promcon.io/2017-munich/). See also the [format documentation](docs/format/README.md). + +A series of blog posts explaining different components of TSDB: +* [The Head Block](https://ganeshvernekar.com/blog/prometheus-tsdb-the-head-block/) +* [WAL and Checkpoint](https://ganeshvernekar.com/blog/prometheus-tsdb-wal-and-checkpoint/) +* [Memory Mapping of Head Chunks from Disk](https://ganeshvernekar.com/blog/prometheus-tsdb-mmapping-head-chunks-from-disk/) +* [Persistent Block and its Index](https://ganeshvernekar.com/blog/prometheus-tsdb-persistent-block-and-its-index/) \ No newline at end of file diff --git a/vendor/github.com/prometheus/prometheus/tsdb/block.go b/vendor/github.com/prometheus/prometheus/tsdb/block.go index 0df30e846c35..3ec2261971e6 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/block.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/block.go @@ -26,6 +26,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -172,7 +173,7 @@ type BlockMetaCompaction struct { // ULIDs of all source head blocks that went into the block. Sources []ulid.ULID `json:"sources,omitempty"` // Indicates that during compaction it resulted in a block without any samples - // so it should be deleted on the next reload. + // so it should be deleted on the next reloadBlocks. Deletable bool `json:"deletable,omitempty"` // Short descriptions of the direct blocks that were used to create // this block. @@ -225,19 +226,14 @@ func writeMetaFile(logger log.Logger, dir string, meta *BlockMeta) (int64, error return 0, err } - var merr tsdb_errors.MultiError n, err := f.Write(jsonMeta) if err != nil { - merr.Add(err) - merr.Add(f.Close()) - return 0, merr.Err() + return 0, tsdb_errors.NewMulti(err, f.Close()).Err() } // Force the kernel to persist the file on disk to avoid data loss if the host crashes. if err := f.Sync(); err != nil { - merr.Add(err) - merr.Add(f.Close()) - return 0, merr.Err() + return 0, tsdb_errors.NewMulti(err, f.Close()).Err() } if err := f.Close(); err != nil { return 0, err @@ -279,10 +275,7 @@ func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, er var closers []io.Closer defer func() { if err != nil { - var merr tsdb_errors.MultiError - merr.Add(err) - merr.Add(closeAll(closers)) - err = merr.Err() + err = tsdb_errors.NewMulti(err, tsdb_errors.CloseAll(closers)).Err() } }() meta, sizeMeta, err := readMetaFile(dir) @@ -332,13 +325,11 @@ func (pb *Block) Close() error { pb.pendingReaders.Wait() - var merr tsdb_errors.MultiError - - merr.Add(pb.chunkr.Close()) - merr.Add(pb.indexr.Close()) - merr.Add(pb.tombstones.Close()) - - return merr.Err() + return tsdb_errors.NewMulti( + pb.chunkr.Close(), + pb.indexr.Close(), + pb.tombstones.Close(), + ).Err() } func (pb *Block) String() string { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go index fa67c1cf4c59..f52b5b932384 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go @@ -14,7 +14,6 @@ package chunkenc import ( - "fmt" "math" "sync" @@ -132,7 +131,7 @@ func (p *pool) Get(e Encoding, b []byte) (Chunk, error) { c.b.count = 0 return c, nil } - return nil, errors.Errorf("invalid encoding %q", e) + return nil, errors.Errorf("invalid chunk encoding %q", e) } func (p *pool) Put(c Chunk) error { @@ -149,7 +148,7 @@ func (p *pool) Put(c Chunk) error { xc.b.count = 0 p.xor.Put(c) default: - return errors.Errorf("invalid encoding %q", c.Encoding()) + return errors.Errorf("invalid chunk encoding %q", c.Encoding()) } return nil } @@ -162,5 +161,5 @@ func FromData(e Encoding, d []byte) (Chunk, error) { case EncXOR: return &XORChunk{b: bstream{count: 0, stream: d}}, nil } - return nil, fmt.Errorf("unknown chunk encoding: %d", e) + return nil, errors.Errorf("invalid chunk encoding %q", e) } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go index 6f7ea09b44f2..11417c38cf1b 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go @@ -27,6 +27,7 @@ import ( "strconv" "github.com/pkg/errors" + "github.com/prometheus/prometheus/tsdb/chunkenc" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" @@ -229,14 +230,13 @@ func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, all } defer func() { if returnErr != nil { - var merr tsdb_errors.MultiError - merr.Add(returnErr) + errs := tsdb_errors.NewMulti(returnErr) if f != nil { - merr.Add(f.Close()) + errs.Add(f.Close()) } // Calling RemoveAll on a non-existent file does not return error. - merr.Add(os.RemoveAll(ptmp)) - returnErr = merr.Err() + errs.Add(os.RemoveAll(ptmp)) + returnErr = errs.Err() } }() if allocSize > 0 { @@ -462,16 +462,16 @@ func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) { } var ( - bs []ByteSlice - cs []io.Closer - merr tsdb_errors.MultiError + bs []ByteSlice + cs []io.Closer ) for _, fn := range files { f, err := fileutil.OpenMmapFile(fn) if err != nil { - merr.Add(errors.Wrap(err, "mmap files")) - merr.Add(closeAll(cs)) - return nil, merr + return nil, tsdb_errors.NewMulti( + errors.Wrap(err, "mmap files"), + tsdb_errors.CloseAll(cs), + ).Err() } cs = append(cs, f) bs = append(bs, realByteSlice(f.Bytes())) @@ -479,15 +479,16 @@ func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) { reader, err := newReader(bs, cs, pool) if err != nil { - merr.Add(err) - merr.Add(closeAll(cs)) - return nil, merr + return nil, tsdb_errors.NewMulti( + err, + tsdb_errors.CloseAll(cs), + ).Err() } return reader, nil } func (s *Reader) Close() error { - return closeAll(s.cs) + return tsdb_errors.CloseAll(s.cs) } // Size returns the size of the chunks. @@ -587,12 +588,3 @@ func sequenceFiles(dir string) ([]string, error) { } return res, nil } - -func closeAll(cs []io.Closer) error { - var merr tsdb_errors.MultiError - - for _, c := range cs { - merr.Add(c.Close()) - } - return merr.Err() -} diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go index 632682218cf2..db45280fd062 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go @@ -27,10 +27,11 @@ import ( "sync" "github.com/pkg/errors" + "go.uber.org/atomic" + "github.com/prometheus/prometheus/tsdb/chunkenc" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" - "go.uber.org/atomic" ) // Head chunk file header fields constants. @@ -152,10 +153,7 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { cdm.closers = map[int]io.Closer{} defer func() { if returnErr != nil { - var merr tsdb_errors.MultiError - merr.Add(returnErr) - merr.Add(closeAllFromMap(cdm.closers)) - returnErr = merr.Err() + returnErr = tsdb_errors.NewMulti(returnErr, closeAllFromMap(cdm.closers)).Err() cdm.mmappedChunkFiles = nil cdm.closers = nil @@ -167,6 +165,11 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { return err } + files, err = repairLastChunkFile(files) + if err != nil { + return err + } + chkFileIndices := make([]int, 0, len(files)) for seq, fn := range files { f, err := fileutil.OpenMmapFile(fn) @@ -226,9 +229,40 @@ func listChunkFiles(dir string) (map[int]string, error) { } res[int(seq)] = filepath.Join(dir, fi.Name()) } + return res, nil } +// repairLastChunkFile deletes the last file if it's empty. +// Because we don't fsync when creating these file, we could end +// up with an empty file at the end during an abrupt shutdown. +func repairLastChunkFile(files map[int]string) (_ map[int]string, returnErr error) { + lastFile := -1 + for seq := range files { + if seq > lastFile { + lastFile = seq + } + } + + if lastFile <= 0 { + return files, nil + } + + info, err := os.Stat(files[lastFile]) + if err != nil { + return files, errors.Wrap(err, "file stat during last head chunk file repair") + } + if info.Size() == 0 { + // Corrupt file, hence remove it. + if err := os.RemoveAll(files[lastFile]); err != nil { + return files, errors.Wrap(err, "delete corrupted, empty head chunk file during last file repair") + } + delete(files, lastFile) + } + + return files, nil +} + // WriteChunk writes the chunk to the disk. // The returned chunk ref is the reference from where the chunk encoding starts for the chunk. func (cdm *ChunkDiskMapper) WriteChunk(seriesRef uint64, mint, maxt int64, chk chunkenc.Chunk) (chkRef uint64, err error) { @@ -333,10 +367,7 @@ func (cdm *ChunkDiskMapper) cut() (returnErr error) { // The file should not be closed if there is no error, // its kept open in the ChunkDiskMapper. if returnErr != nil { - var merr tsdb_errors.MultiError - merr.Add(returnErr) - merr.Add(newFile.Close()) - returnErr = merr.Err() + returnErr = tsdb_errors.NewMulti(returnErr, newFile.Close()).Err() } }() @@ -680,13 +711,13 @@ func (cdm *ChunkDiskMapper) Truncate(mint int64) error { } cdm.readPathMtx.RUnlock() - var merr tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() // Cut a new file only if the current file has some chunks. if cdm.curFileSize() > HeadChunkFileHeaderSize { - merr.Add(cdm.CutNewFile()) + errs.Add(cdm.CutNewFile()) } - merr.Add(cdm.deleteFiles(removedFiles)) - return merr.Err() + errs.Add(cdm.deleteFiles(removedFiles)) + return errs.Err() } func (cdm *ChunkDiskMapper) deleteFiles(removedFiles []int) error { @@ -758,23 +789,23 @@ func (cdm *ChunkDiskMapper) Close() error { } cdm.closed = true - var merr tsdb_errors.MultiError - merr.Add(closeAllFromMap(cdm.closers)) - merr.Add(cdm.finalizeCurFile()) - merr.Add(cdm.dir.Close()) - + errs := tsdb_errors.NewMulti( + closeAllFromMap(cdm.closers), + cdm.finalizeCurFile(), + cdm.dir.Close(), + ) cdm.mmappedChunkFiles = map[int]*mmappedChunkFile{} cdm.closers = map[int]io.Closer{} - return merr.Err() + return errs.Err() } func closeAllFromMap(cs map[int]io.Closer) error { - var merr tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for _, c := range cs { - merr.Add(c.Close()) + errs.Add(c.Close()) } - return merr.Err() + return errs.Err() } const inBufferShards = 128 // 128 is a randomly chosen number. diff --git a/vendor/github.com/prometheus/prometheus/tsdb/compact.go b/vendor/github.com/prometheus/prometheus/tsdb/compact.go index 74f54fdb9c81..9026b37bf26f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/compact.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/compact.go @@ -29,6 +29,7 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -450,17 +451,16 @@ func (c *LeveledCompactor) Compact(dest string, dirs []string, open []*Block) (u return uid, nil } - var merr tsdb_errors.MultiError - merr.Add(err) + errs := tsdb_errors.NewMulti(err) if err != context.Canceled { for _, b := range bs { if err := b.setCompactionFailed(); err != nil { - merr.Add(errors.Wrapf(err, "setting compaction failed for block: %s", b.Dir())) + errs.Add(errors.Wrapf(err, "setting compaction failed for block: %s", b.Dir())) } } } - return uid, merr + return uid, errs.Err() } func (c *LeveledCompactor) Write(dest string, b BlockReader, mint, maxt int64, parent *BlockMeta) (ulid.ULID, error) { @@ -488,6 +488,12 @@ func (c *LeveledCompactor) Write(dest string, b BlockReader, mint, maxt int64, p } if meta.Stats.NumSamples == 0 { + level.Info(c.logger).Log( + "msg", "write block resulted in empty block", + "mint", meta.MinTime, + "maxt", meta.MaxTime, + "duration", time.Since(start), + ) return ulid.ULID{}, nil } @@ -527,10 +533,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe tmp := dir + tmpForCreationBlockDirSuffix var closers []io.Closer defer func(t time.Time) { - var merr tsdb_errors.MultiError - merr.Add(err) - merr.Add(closeAll(closers)) - err = merr.Err() + err = tsdb_errors.NewMulti(err, tsdb_errors.CloseAll(closers)).Err() // RemoveAll returns no error when tmp doesn't exist so it is safe to always run it. if err := os.RemoveAll(tmp); err != nil { @@ -587,13 +590,13 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe // though these are covered under defer. This is because in Windows, // you cannot delete these unless they are closed and the defer is to // make sure they are closed if the function exits due to an error above. - var merr tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for _, w := range closers { - merr.Add(w.Close()) + errs.Add(w.Close()) } closers = closers[:0] // Avoid closing the writers twice in the defer. - if merr.Err() != nil { - return merr.Err() + if errs.Err() != nil { + return errs.Err() } // Populated block is empty, so exit early. @@ -653,12 +656,11 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, overlapping bool ) defer func() { - var merr tsdb_errors.MultiError - merr.Add(err) - if cerr := closeAll(closers); cerr != nil { - merr.Add(errors.Wrap(cerr, "close")) + errs := tsdb_errors.NewMulti(err) + if cerr := tsdb_errors.CloseAll(closers); cerr != nil { + errs.Add(errors.Wrap(cerr, "close")) } - err = merr.Err() + err = errs.Err() c.metrics.populatingBlocks.Set(0) }() c.metrics.populatingBlocks.Set(1) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/db.go b/vendor/github.com/prometheus/prometheus/tsdb/db.go index 4c70a4052773..81669829de04 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/db.go @@ -34,16 +34,15 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/errgroup" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" + _ "github.com/prometheus/prometheus/tsdb/goversion" // Load the package into main to make sure minium Go version is met. "github.com/prometheus/prometheus/tsdb/wal" - "golang.org/x/sync/errgroup" - - // Load the package into main to make sure minium Go version is met. - _ "github.com/prometheus/prometheus/tsdb/goversion" ) const ( @@ -199,7 +198,7 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { }) m.symbolTableSize = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "prometheus_tsdb_symbol_table_size_bytes", - Help: "Size of symbol table on disk (in bytes)", + Help: "Size of symbol table in memory for loaded blocks", }, func() float64 { db.mtx.RLock() blocks := db.blocks[:] @@ -216,7 +215,7 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { }) m.reloadsFailed = prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_reloads_failures_total", - Help: "Number of times the database failed to reload block data from disk.", + Help: "Number of times the database failed to reloadBlocks block data from disk.", }) m.compactionsTriggered = prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_compactions_triggered_total", @@ -334,10 +333,10 @@ func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) { return err } defer func() { - var merr tsdb_errors.MultiError - merr.Add(returnErr) - merr.Add(errors.Wrap(head.Close(), "closing Head")) - returnErr = merr.Err() + returnErr = tsdb_errors.NewMulti( + returnErr, + errors.Wrap(head.Close(), "closing Head"), + ).Err() }() // Set the min valid time for the ingested wal samples // to be no lower than the maxt of the last block. @@ -467,11 +466,11 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) { level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", b) } } - var merr tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for ulid, err := range corrupted { - merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + errs.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) } - return nil, merr.Err() + return nil, errs.Err() } if len(loadable) == 0 { @@ -515,12 +514,7 @@ func (db *DBReadOnly) Close() error { } close(db.closed) - var merr tsdb_errors.MultiError - - for _, b := range db.closers { - merr.Add(b.Close()) - } - return merr.Err() + return tsdb_errors.CloseAll(db.closers) } // Open returns a new DB in the given directory. If options are empty, DefaultOptions will be used. @@ -553,7 +547,7 @@ func validateOpts(opts *Options, rngs []int64) (*Options, []int64) { return opts, rngs } -func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs []int64) (db *DB, err error) { +func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs []int64) (_ *DB, returnedErr error) { if err := os.MkdirAll(dir, 0777); err != nil { return nil, err } @@ -584,7 +578,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs return nil, errors.Wrap(err, "remove tmp dirs") } - db = &DB{ + db := &DB{ dir: dir, logger: l, opts: opts, @@ -595,6 +589,20 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs chunkPool: chunkenc.NewPool(), blocksToDelete: opts.BlocksToDelete, } + defer func() { + // Close files if startup fails somewhere. + if returnedErr == nil { + return + } + + close(db.donec) // DB is never run if it was an error, so close this channel here. + + returnedErr = tsdb_errors.NewMulti( + returnedErr, + errors.Wrap(db.Close(), "close DB after failed startup"), + ).Err() + }() + if db.blocksToDelete == nil { db.blocksToDelete = DefaultBlocksToDelete(db) } @@ -611,6 +619,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs db.lockf = lockf } + var err error ctx, cancel := context.WithCancel(context.Background()) db.compactor, err = NewLeveledCompactor(ctx, r, l, rngs, db.chunkPool) if err != nil { @@ -769,20 +778,26 @@ func (a dbAppender) Commit() error { } // Compact data if possible. After successful compaction blocks are reloaded -// which will also trigger blocks to be deleted that fall out of the retention -// window. -// If no blocks are compacted, the retention window state doesn't change. Thus, -// this is sufficient to reliably delete old data. -// Old blocks are only deleted on reload based on the new block's parent information. -// See DB.reload documentation for further information. -func (db *DB) Compact() (err error) { +// which will also delete the blocks that fall out of the retention window. +// Old blocks are only deleted on reloadBlocks based on the new block's parent information. +// See DB.reloadBlocks documentation for further information. +func (db *DB) Compact() (returnErr error) { db.cmtx.Lock() defer db.cmtx.Unlock() defer func() { - if err != nil { + if returnErr != nil { db.metrics.compactionsFailed.Inc() } }() + + lastBlockMaxt := int64(math.MinInt64) + defer func() { + returnErr = tsdb_errors.NewMulti( + returnErr, + errors.Wrap(db.head.truncateWAL(lastBlockMaxt), "WAL truncation in Compact defer"), + ).Err() + }() + // Check whether we have pending head blocks that are ready to be persisted. // They have the highest priority. for { @@ -804,55 +819,59 @@ func (db *DB) Compact() (err error) { // so in order to make sure that overlaps are evaluated // consistently, we explicitly remove the last value // from the block interval here. - head := NewRangeHead(db.head, mint, maxt-1) - if err := db.compactHead(head); err != nil { - return err + if err := db.compactHead(NewRangeHead(db.head, mint, maxt-1)); err != nil { + return errors.Wrap(err, "compact head") } + // Consider only successful compactions for WAL truncation. + lastBlockMaxt = maxt + } + + // Clear some disk space before compacting blocks, especially important + // when Head compaction happened over a long time range. + if err := db.head.truncateWAL(lastBlockMaxt); err != nil { + return errors.Wrap(err, "WAL truncation in Compact") } return db.compactBlocks() } -// CompactHead compacts the given the RangeHead. -func (db *DB) CompactHead(head *RangeHead) (err error) { +// CompactHead compacts the given RangeHead. +func (db *DB) CompactHead(head *RangeHead) error { db.cmtx.Lock() defer db.cmtx.Unlock() - return db.compactHead(head) + if err := db.compactHead(head); err != nil { + return errors.Wrap(err, "compact head") + } + + if err := db.head.truncateWAL(head.BlockMaxTime()); err != nil { + return errors.Wrap(err, "WAL truncation") + } + return nil } -// compactHead compacts the given the RangeHead. +// compactHead compacts the given RangeHead. // The compaction mutex should be held before calling this method. -func (db *DB) compactHead(head *RangeHead) (err error) { - // Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime). - // Because of this block intervals are always +1 than the total samples it includes. - maxt := head.MaxTime() + 1 - uid, err := db.compactor.Write(db.dir, head, head.MinTime(), maxt, nil) +func (db *DB) compactHead(head *RangeHead) error { + uid, err := db.compactor.Write(db.dir, head, head.MinTime(), head.BlockMaxTime(), nil) if err != nil { return errors.Wrap(err, "persist head block") } runtime.GC() - - if err := db.reload(); err != nil { + if err := db.reloadBlocks(); err != nil { if errRemoveAll := os.RemoveAll(filepath.Join(db.dir, uid.String())); errRemoveAll != nil { - var merr tsdb_errors.MultiError - merr.Add(errors.Wrap(err, "reload blocks")) - merr.Add(errors.Wrapf(errRemoveAll, "delete persisted head block after failed db reload:%s", uid)) - return merr.Err() + return tsdb_errors.NewMulti( + errors.Wrap(err, "reloadBlocks blocks"), + errors.Wrapf(errRemoveAll, "delete persisted head block after failed db reloadBlocks:%s", uid), + ).Err() } - return errors.Wrap(err, "reload blocks") + return errors.Wrap(err, "reloadBlocks blocks") } - if (uid == ulid.ULID{}) { - // Compaction resulted in an empty block. - // Head truncating during db.reload() depends on the persisted blocks and - // in this case no new block will be persisted so manually truncate the head. - if err = db.head.Truncate(maxt); err != nil { - return errors.Wrap(err, "head truncate failed (in compact)") - } + if err = db.head.truncateMemory(head.BlockMaxTime()); err != nil { + return errors.Wrap(err, "head memory truncate") } runtime.GC() - return nil } @@ -881,11 +900,11 @@ func (db *DB) compactBlocks() (err error) { } runtime.GC() - if err := db.reload(); err != nil { + if err := db.reloadBlocks(); err != nil { if err := os.RemoveAll(filepath.Join(db.dir, uid.String())); err != nil { - return errors.Wrapf(err, "delete compacted block after failed db reload:%s", uid) + return errors.Wrapf(err, "delete compacted block after failed db reloadBlocks:%s", uid) } - return errors.Wrap(err, "reload blocks") + return errors.Wrap(err, "reloadBlocks blocks") } runtime.GC() } @@ -904,9 +923,23 @@ func getBlock(allBlocks []*Block, id ulid.ULID) (*Block, bool) { return nil, false } -// reload blocks and trigger head truncation if new blocks appeared. +// reload reloads blocks and truncates the head and its WAL. +func (db *DB) reload() error { + if err := db.reloadBlocks(); err != nil { + return errors.Wrap(err, "reloadBlocks") + } + if len(db.blocks) == 0 { + return nil + } + if err := db.head.Truncate(db.blocks[len(db.blocks)-1].MaxTime()); err != nil { + return errors.Wrap(err, "head truncate") + } + return nil +} + +// reloadBlocks reloads blocks without touching head. // Blocks that are obsolete due to replacement or retention will be deleted. -func (db *DB) reload() (err error) { +func (db *DB) reloadBlocks() (err error) { defer func() { if err != nil { db.metrics.reloadsFailed.Inc() @@ -945,11 +978,11 @@ func (db *DB) reload() (err error) { block.Close() } } - var merr tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for ulid, err := range corrupted { - merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + errs.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) } - return merr.Err() + return errs.Err() } var ( @@ -989,7 +1022,7 @@ func (db *DB) reload() (err error) { blockMetas = append(blockMetas, b.Meta()) } if overlaps := OverlappingBlocks(blockMetas); len(overlaps) > 0 { - level.Warn(db.logger).Log("msg", "Overlapping blocks found during reload", "detail", overlaps.String()) + level.Warn(db.logger).Log("msg", "Overlapping blocks found during reloadBlocks", "detail", overlaps.String()) } // Append blocks to old, deletable blocks, so we can close them. @@ -999,15 +1032,9 @@ func (db *DB) reload() (err error) { } } if err := db.deleteBlocks(deletable); err != nil { - return err - } - - // Garbage collect data in the head if the most recent persisted block - // covers data of its current time range. - if len(toLoad) == 0 { - return nil + return errors.Wrapf(err, "delete %v blocks", len(deletable)) } - return errors.Wrap(db.head.Truncate(toLoad[len(toLoad)-1].Meta().MaxTime), "head truncate failed") + return nil } func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Pool) (blocks []*Block, corrupted map[ulid.ULID]error, err error) { @@ -1020,7 +1047,7 @@ func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Po for _, bDir := range bDirs { meta, _, err := readMetaFile(bDir) if err != nil { - level.Error(l).Log("msg", "Failed to read meta.json for a block during reload. Skipping", "dir", bDir, "err", err) + level.Error(l).Log("msg", "Failed to read meta.json for a block during reloadBlocks. Skipping", "dir", bDir, "err", err) continue } @@ -1077,7 +1104,7 @@ func deletableBlocks(db *DB, blocks []*Block) map[ulid.ULID]struct{} { // set in the db options. func BeyondTimeRetention(db *DB, blocks []*Block) (deletable map[ulid.ULID]struct{}) { // Time retention is disabled or no blocks to work with. - if len(db.blocks) == 0 || db.opts.RetentionDuration == 0 { + if len(blocks) == 0 || db.opts.RetentionDuration == 0 { return } @@ -1100,7 +1127,7 @@ func BeyondTimeRetention(db *DB, blocks []*Block) (deletable map[ulid.ULID]struc // set in the db options. func BeyondSizeRetention(db *DB, blocks []*Block) (deletable map[ulid.ULID]struct{}) { // Size retention is disabled or no blocks to work with. - if len(db.blocks) == 0 || db.opts.MaxBytes <= 0 { + if len(blocks) == 0 || db.opts.MaxBytes <= 0 { return } @@ -1295,7 +1322,9 @@ func (db *DB) Head() *Head { // Close the partition. func (db *DB) Close() error { close(db.stopc) - db.compactCancel() + if db.compactCancel != nil { + db.compactCancel() + } <-db.donec db.mtx.Lock() @@ -1308,15 +1337,14 @@ func (db *DB) Close() error { g.Go(pb.Close) } - var merr tsdb_errors.MultiError - - merr.Add(g.Wait()) - + errs := tsdb_errors.NewMulti(g.Wait()) if db.lockf != nil { - merr.Add(db.lockf.Release()) + errs.Add(db.lockf.Release()) + } + if db.head != nil { + errs.Add(db.head.Close()) } - merr.Add(db.head.Close()) - return merr.Err() + return errs.Err() } // DisableCompactions disables auto compactions. @@ -1502,7 +1530,11 @@ func (db *DB) CleanTombstones() (err error) { newUIDs = append(newUIDs, *uid) } } - return errors.Wrap(db.reload(), "reload blocks") + + if err := db.reloadBlocks(); err != nil { + return errors.Wrap(err, "reload blocks") + } + return nil } func isBlockDir(fi os.FileInfo) bool { @@ -1577,15 +1609,6 @@ func nextSequenceFile(dir string) (string, int, error) { return filepath.Join(dir, fmt.Sprintf("%0.6d", i+1)), int(i + 1), nil } -func closeAll(cs []io.Closer) error { - var merr tsdb_errors.MultiError - - for _, c := range cs { - merr.Add(c.Close()) - } - return merr.Err() -} - func exponential(d, min, max time.Duration) time.Duration { d *= 2 if d < min { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go b/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go index 69d36624800f..aeac4d27716d 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go @@ -17,21 +17,59 @@ package errors import ( "bytes" "fmt" + "io" ) -// The MultiError type implements the error interface, and contains the -// Errors used to construct it. -type MultiError []error +// multiError type allows combining multiple errors into one. +type multiError []error -// Returns a concatenated string of the contained errors -func (es MultiError) Error() string { +// NewMulti returns multiError with provided errors added if not nil. +func NewMulti(errs ...error) multiError { // nolint:golint + m := multiError{} + m.Add(errs...) + return m +} + +// Add adds single or many errors to the error list. Each error is added only if not nil. +// If the error is a nonNilMultiError type, the errors inside nonNilMultiError are added to the main multiError. +func (es *multiError) Add(errs ...error) { + for _, err := range errs { + if err == nil { + continue + } + if merr, ok := err.(nonNilMultiError); ok { + *es = append(*es, merr.errs...) + continue + } + *es = append(*es, err) + } +} + +// Err returns the error list as an error or nil if it is empty. +func (es multiError) Err() error { + if len(es) == 0 { + return nil + } + return nonNilMultiError{errs: es} +} + +// nonNilMultiError implements the error interface, and it represents +// multiError with at least one error inside it. +// This type is needed to make sure that nil is returned when no error is combined in multiError for err != nil +// check to work. +type nonNilMultiError struct { + errs multiError +} + +// Error returns a concatenated string of the contained errors. +func (es nonNilMultiError) Error() string { var buf bytes.Buffer - if len(es) > 1 { - fmt.Fprintf(&buf, "%d errors: ", len(es)) + if len(es.errs) > 1 { + fmt.Fprintf(&buf, "%d errors: ", len(es.errs)) } - for i, err := range es { + for i, err := range es.errs { if i != 0 { buf.WriteString("; ") } @@ -41,22 +79,11 @@ func (es MultiError) Error() string { return buf.String() } -// Add adds the error to the error list if it is not nil. -func (es *MultiError) Add(err error) { - if err == nil { - return - } - if merr, ok := err.(MultiError); ok { - *es = append(*es, merr...) - } else { - *es = append(*es, err) - } -} - -// Err returns the error list as an error or nil if it is empty. -func (es MultiError) Err() error { - if len(es) == 0 { - return nil +// CloseAll closes all given closers while recording error in MultiError. +func CloseAll(cs []io.Closer) error { + errs := NewMulti() + for _, c := range cs { + errs.Add(c.Close()) } - return es + return errs.Err() } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go index 5cb123810392..4dbca4f9740f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go @@ -48,7 +48,7 @@ func OpenMmapFileWithSize(path string, size int) (mf *MmapFile, retErr error) { b, err := mmap(f, size) if err != nil { - return nil, errors.Wrap(err, "mmap") + return nil, errors.Wrapf(err, "mmap, size %d", size) } return &MmapFile{f: f, b: b}, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head.go b/vendor/github.com/prometheus/prometheus/tsdb/head.go index 3a577763d64a..dad122778d95 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head.go @@ -28,6 +28,8 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -38,7 +40,6 @@ import ( "github.com/prometheus/prometheus/tsdb/tombstones" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wal" - "go.uber.org/atomic" ) var ( @@ -52,11 +53,12 @@ var ( // Head handles reads and writes of time series data within a time window. type Head struct { - chunkRange atomic.Int64 - numSeries atomic.Uint64 - minTime, maxTime atomic.Int64 // Current min and max of the samples included in the head. - minValidTime atomic.Int64 // Mint allowed to be added to the head. It shouldn't be lower than the maxt of the last persisted block. - lastSeriesID atomic.Uint64 + chunkRange atomic.Int64 + numSeries atomic.Uint64 + minTime, maxTime atomic.Int64 // Current min and max of the samples included in the head. + minValidTime atomic.Int64 // Mint allowed to be added to the head. It shouldn't be lower than the maxt of the last persisted block. + lastWALTruncationTime atomic.Int64 + lastSeriesID atomic.Uint64 metrics *headMetrics wal *wal.WAL @@ -323,6 +325,7 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, chunkRange int h.chunkRange.Store(chunkRange) h.minTime.Store(math.MaxInt64) h.maxTime.Store(math.MinInt64) + h.lastWALTruncationTime.Store(math.MinInt64) h.metrics = newHeadMetrics(h, r) if pool == nil { @@ -776,8 +779,20 @@ func (h *Head) removeCorruptedMmappedChunks(err error) map[uint64][]*mmappedChun return mmappedChunks } -// Truncate removes old data before mint from the head. +// Truncate removes old data before mint from the head and WAL. func (h *Head) Truncate(mint int64) (err error) { + initialize := h.MinTime() == math.MaxInt64 + if err := h.truncateMemory(mint); err != nil { + return err + } + if initialize { + return nil + } + return h.truncateWAL(mint) +} + +// truncateMemory removes old data before mint from the head. +func (h *Head) truncateMemory(mint int64) (err error) { defer func() { if err != nil { h.metrics.headTruncateFail.Inc() @@ -813,11 +828,16 @@ func (h *Head) Truncate(mint int64) (err error) { if err := h.chunkDiskMapper.Truncate(mint); err != nil { return errors.Wrap(err, "truncate chunks.HeadReadWriter") } + return nil +} - if h.wal == nil { +// truncateWAL removes old data before mint from the WAL. +func (h *Head) truncateWAL(mint int64) error { + if h.wal == nil || mint <= h.lastWALTruncationTime.Load() { return nil } - start = time.Now() + start := time.Now() + h.lastWALTruncationTime.Store(mint) first, last, err := wal.Segments(h.wal.Dir()) if err != nil { @@ -825,8 +845,7 @@ func (h *Head) Truncate(mint int64) (err error) { } // Start a new segment, so low ingestion volume TSDB don't have more WAL than // needed. - err = h.wal.NextSegment() - if err != nil { + if err := h.wal.NextSegment(); err != nil { return errors.Wrap(err, "next segment") } last-- // Never consider last segment for checkpoint. @@ -950,10 +969,19 @@ func (h *RangeHead) MinTime() int64 { return h.mint } +// MaxTime returns the max time of actual data fetch-able from the head. +// This controls the chunks time range which is closed [b.MinTime, b.MaxTime]. func (h *RangeHead) MaxTime() int64 { return h.maxt } +// BlockMaxTime returns the max time of the potential block created from this head. +// It's different to MaxTime as we need to add +1 millisecond to block maxt because block +// intervals are half-open: [b.MinTime, b.MaxTime). Block intervals are always +1 than the total samples it includes. +func (h *RangeHead) BlockMaxTime() int64 { + return h.MaxTime() + 1 +} + func (h *RangeHead) NumSeries() uint64 { return h.head.NumSeries() } @@ -1437,12 +1465,11 @@ func (h *Head) Close() error { h.closedMtx.Lock() defer h.closedMtx.Unlock() h.closed = true - var merr tsdb_errors.MultiError - merr.Add(h.chunkDiskMapper.Close()) + errs := tsdb_errors.NewMulti(h.chunkDiskMapper.Close()) if h.wal != nil { - merr.Add(h.wal.Close()) + errs.Add(h.wal.Close()) } - return merr.Err() + return errs.Err() } type headChunkReader struct { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go index d7559b346ac6..c747f5eeed37 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go @@ -29,6 +29,7 @@ import ( "unsafe" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/encoding" @@ -1074,10 +1075,10 @@ func NewFileReader(path string) (*Reader, error) { } r, err := newReader(realByteSlice(f.Bytes()), f) if err != nil { - var merr tsdb_errors.MultiError - merr.Add(err) - merr.Add(f.Close()) - return nil, merr + return nil, tsdb_errors.NewMulti( + err, + f.Close(), + ).Err() } return r, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index 6df8cba9b4eb..f666bb790447 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -20,6 +20,7 @@ import ( "unicode/utf8" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -96,12 +97,14 @@ func (q *blockBaseQuerier) Close() error { if q.closed { return errors.New("block querier already closed") } - var merr tsdb_errors.MultiError - merr.Add(q.index.Close()) - merr.Add(q.chunks.Close()) - merr.Add(q.tombstones.Close()) + + errs := tsdb_errors.NewMulti( + q.index.Close(), + q.chunks.Close(), + q.tombstones.Close(), + ) q.closed = true - return merr.Err() + return errs.Err() } type blockQuerier struct { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/record/record.go b/vendor/github.com/prometheus/prometheus/tsdb/record/record.go index e7df28766c51..408882e83243 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/record/record.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/record/record.go @@ -19,6 +19,7 @@ import ( "sort" "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/tombstones" diff --git a/vendor/github.com/prometheus/prometheus/tsdb/repair.go b/vendor/github.com/prometheus/prometheus/tsdb/repair.go index 02114cd4f67b..cc777546e5c0 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/repair.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/repair.go @@ -23,6 +23,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" ) @@ -82,18 +83,18 @@ func repairBadIndexVersion(logger log.Logger, dir string) error { return errors.Wrapf(err, "copy content of index to index.repaired for block dir: %v", d) } - var merr tsdb_errors.MultiError - // Set the 5th byte to 2 to indicate the correct file format version. if _, err := repl.WriteAt([]byte{2}, 4); err != nil { - merr.Add(errors.Wrap(err, "rewrite of index.repaired")) - merr.Add(errors.Wrap(repl.Close(), "close")) - return errors.Wrapf(merr.Err(), "block dir: %v", d) + return tsdb_errors.NewMulti( + errors.Wrapf(err, "rewrite of index.repaired for block dir: %v", d), + errors.Wrap(repl.Close(), "close"), + ).Err() } if err := repl.Sync(); err != nil { - merr.Add(errors.Wrap(err, "sync of index.repaired")) - merr.Add(errors.Wrap(repl.Close(), "close")) - return errors.Wrapf(merr.Err(), "block dir: %v", d) + return tsdb_errors.NewMulti( + errors.Wrapf(err, "sync of index.repaired for block dir: %v", d), + errors.Wrap(repl.Close(), "close"), + ).Err() } if err := repl.Close(); err != nil { return errors.Wrapf(repl.Close(), "close repaired index for block dir: %v", d) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go b/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go index d34a75af4d49..aa74a7d1e356 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go @@ -27,6 +27,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "github.com/prometheus/prometheus/tsdb/encoding" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" @@ -125,10 +126,8 @@ func WriteFile(logger log.Logger, dir string, tr Reader) (int64, error) { } size += n - var merr tsdb_errors.MultiError - if merr.Add(f.Sync()); merr.Err() != nil { - merr.Add(f.Close()) - return 0, merr.Err() + if err := f.Sync(); err != nil { + return 0, tsdb_errors.NewMulti(err, f.Close()).Err() } if err = f.Close(); err != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go b/vendor/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go index 15f129d318df..be2c63f9f432 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go @@ -19,6 +19,7 @@ import ( "path/filepath" "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/storage" ) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wal.go b/vendor/github.com/prometheus/prometheus/tsdb/wal.go index 1e503a1481b5..7856ac14c657 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wal.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wal.go @@ -31,6 +31,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/fileutil" diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wal/checkpoint.go b/vendor/github.com/prometheus/prometheus/tsdb/wal/checkpoint.go index 33e2e58cedbf..a264e1e958eb 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wal/checkpoint.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wal/checkpoint.go @@ -28,6 +28,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/record" @@ -67,14 +68,12 @@ func DeleteCheckpoints(dir string, maxIndex int) error { return err } - var errs tsdb_errors.MultiError + errs := tsdb_errors.NewMulti() for _, checkpoint := range checkpoints { if checkpoint.index >= maxIndex { break } - if err := os.RemoveAll(filepath.Join(dir, checkpoint.name)); err != nil { - errs.Add(err) - } + errs.Add(os.RemoveAll(filepath.Join(dir, checkpoint.name))) } return errs.Err() } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go b/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go index c5401ddf1e8e..3aa87b2c0706 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go @@ -33,6 +33,7 @@ import ( "github.com/golang/snappy" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/tsdb/fileutil" ) @@ -73,9 +74,18 @@ func (p *page) reset() { p.flushed = 0 } +// SegmentFile represents the underlying file used to store a segment. +type SegmentFile interface { + Stat() (os.FileInfo, error) + Sync() error + io.Writer + io.Reader + io.Closer +} + // Segment represents a segment file. type Segment struct { - *os.File + SegmentFile dir string i int } @@ -129,7 +139,7 @@ func OpenWriteSegment(logger log.Logger, dir string, k int) (*Segment, error) { return nil, errors.Wrap(err, "zero-pad torn page") } } - return &Segment{File: f, i: k, dir: dir}, nil + return &Segment{SegmentFile: f, i: k, dir: dir}, nil } // CreateSegment creates a new segment k in dir. @@ -138,7 +148,7 @@ func CreateSegment(dir string, k int) (*Segment, error) { if err != nil { return nil, err } - return &Segment{File: f, i: k, dir: dir}, nil + return &Segment{SegmentFile: f, i: k, dir: dir}, nil } // OpenReadSegment opens the segment with the given filename. @@ -151,7 +161,7 @@ func OpenReadSegment(fn string) (*Segment, error) { if err != nil { return nil, err } - return &Segment{File: f, i: k, dir: filepath.Dir(fn)}, nil + return &Segment{SegmentFile: f, i: k, dir: filepath.Dir(fn)}, nil } // WAL is a write ahead log that stores records in segment files. @@ -516,8 +526,10 @@ func (w *WAL) flushPage(clear bool) error { if clear { p.alloc = pageSize // Write till end of page. } + n, err := w.segment.Write(p.buf[p.flushed:p.alloc]) if err != nil { + p.flushed += n return err } p.flushed += n @@ -663,6 +675,9 @@ func (w *WAL) log(rec []byte, final bool) error { if w.page.full() { if err := w.flushPage(true); err != nil { + // TODO When the flushing fails at this point and the record has not been + // fully written to the buffer, we end up with a corrupted WAL because some part of the + // record have been written to the buffer, while the rest of the record will be discarded. return err } } @@ -704,7 +719,7 @@ func (w *WAL) Truncate(i int) (err error) { func (w *WAL) fsync(f *Segment) error { start := time.Now() - err := f.File.Sync() + err := f.Sync() w.metrics.fsyncDuration.Observe(time.Since(start).Seconds()) return err } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wal/watcher.go b/vendor/github.com/prometheus/prometheus/tsdb/wal/watcher.go index 1fb78005f8c5..8670567f132b 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wal/watcher.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wal/watcher.go @@ -29,6 +29,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/tsdb/record" ) diff --git a/vendor/github.com/prometheus/prometheus/util/testutil/directory.go b/vendor/github.com/prometheus/prometheus/util/testutil/directory.go index 4f0f08fe8f34..1c87e35de5d7 100644 --- a/vendor/github.com/prometheus/prometheus/util/testutil/directory.go +++ b/vendor/github.com/prometheus/prometheus/util/testutil/directory.go @@ -21,6 +21,8 @@ import ( "path/filepath" "strconv" "testing" + + "github.com/stretchr/testify/require" ) const ( @@ -137,32 +139,32 @@ func NewTemporaryDirectory(name string, t T) (handler TemporaryDirectory) { func DirHash(t *testing.T, path string) []byte { hash := sha256.New() err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - Ok(t, err) + require.NoError(t, err) if info.IsDir() { return nil } f, err := os.Open(path) - Ok(t, err) + require.NoError(t, err) defer f.Close() _, err = io.Copy(hash, f) - Ok(t, err) + require.NoError(t, err) _, err = io.WriteString(hash, strconv.Itoa(int(info.Size()))) - Ok(t, err) + require.NoError(t, err) _, err = io.WriteString(hash, info.Name()) - Ok(t, err) + require.NoError(t, err) modTime, err := info.ModTime().GobEncode() - Ok(t, err) + require.NoError(t, err) _, err = io.WriteString(hash, string(modTime)) - Ok(t, err) + require.NoError(t, err) return nil }) - Ok(t, err) + require.NoError(t, err) return hash.Sum(nil) } diff --git a/vendor/github.com/prometheus/prometheus/util/testutil/testing.go b/vendor/github.com/prometheus/prometheus/util/testutil/testing.go index 1645f80d5c35..8ec50cb00f42 100644 --- a/vendor/github.com/prometheus/prometheus/util/testutil/testing.go +++ b/vendor/github.com/prometheus/prometheus/util/testutil/testing.go @@ -23,140 +23,11 @@ package testutil import ( - "fmt" - "reflect" "testing" - "github.com/davecgh/go-spew/spew" - "github.com/pmezard/go-difflib/difflib" "go.uber.org/goleak" ) -// This package is imported by non-test code and therefore cannot import the -// testing package, which has side effects such as adding flags. Hence we use an -// interface to testing.{T,B}. -type TB interface { - Helper() - Fatalf(string, ...interface{}) -} - -// Assert fails the test if the condition is false. -func Assert(tb TB, condition bool, format string, a ...interface{}) { - tb.Helper() - if !condition { - tb.Fatalf("\033[31m"+format+"\033[39m\n", a...) - } -} - -// Ok fails the test if an err is not nil. -func Ok(tb TB, err error) { - tb.Helper() - if err != nil { - tb.Fatalf("\033[31munexpected error: %v\033[39m\n", err) - } -} - -// NotOk fails the test if an err is nil. -func NotOk(tb TB, err error, a ...interface{}) { - tb.Helper() - if err == nil { - if len(a) != 0 { - format := a[0].(string) - tb.Fatalf("\033[31m"+format+": expected error, got none\033[39m", a[1:]...) - } - tb.Fatalf("\033[31mexpected error, got none\033[39m") - } -} - -// Equals fails the test if exp is not equal to act. -func Equals(tb TB, exp, act interface{}, msgAndArgs ...interface{}) { - tb.Helper() - if !reflect.DeepEqual(exp, act) { - tb.Fatalf("\033[31m%s\n\nexp: %#v\n\ngot: %#v%s\033[39m\n", formatMessage(msgAndArgs), exp, act, diff(exp, act)) - } -} - -func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { - t := reflect.TypeOf(v) - k := t.Kind() - - if k == reflect.Ptr { - t = t.Elem() - k = t.Kind() - } - return t, k -} - -// diff returns a diff of both values as long as both are of the same type and -// are a struct, map, slice, array or string. Otherwise it returns an empty string. -func diff(expected interface{}, actual interface{}) string { - if expected == nil || actual == nil { - return "" - } - - et, ek := typeAndKind(expected) - at, _ := typeAndKind(actual) - if et != at { - return "" - } - - if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array && ek != reflect.String { - return "" - } - - var e, a string - c := spew.ConfigState{ - Indent: " ", - DisablePointerAddresses: true, - DisableCapacities: true, - SortKeys: true, - } - if et != reflect.TypeOf("") { - e = c.Sdump(expected) - a = c.Sdump(actual) - } else { - e = reflect.ValueOf(expected).String() - a = reflect.ValueOf(actual).String() - } - - diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ - A: difflib.SplitLines(e), - B: difflib.SplitLines(a), - FromFile: "Expected", - FromDate: "", - ToFile: "Actual", - ToDate: "", - Context: 1, - }) - return "\n\nDiff:\n" + diff -} - -// ErrorEqual compares Go errors for equality. -func ErrorEqual(tb TB, left, right error, msgAndArgs ...interface{}) { - tb.Helper() - if left == right { - return - } - - if left != nil && right != nil { - Equals(tb, left.Error(), right.Error(), msgAndArgs...) - return - } - - tb.Fatalf("\033[31m%s\n\nexp: %#v\n\ngot: %#v\033[39m\n", formatMessage(msgAndArgs), left, right) -} - -func formatMessage(msgAndArgs []interface{}) string { - if len(msgAndArgs) == 0 { - return "" - } - - if msg, ok := msgAndArgs[0].(string); ok { - return fmt.Sprintf("\n\nmsg: "+msg, msgAndArgs[1:]...) - } - return "" -} - // TolerantVerifyLeak verifies go leaks but excludes the go routines that are // launched as side effects of some of our dependencies. func TolerantVerifyLeak(m *testing.M) { diff --git a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go index 284278e5277e..ea4a2c3b1f4a 100644 --- a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go +++ b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go @@ -37,6 +37,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/gate" "github.com/prometheus/prometheus/pkg/labels" diff --git a/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go b/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go index bd83f6e2fbc2..e058e22172a8 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go +++ b/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go @@ -24,14 +24,15 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" "github.com/prometheus/prometheus/tsdb" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" + "golang.org/x/sync/errgroup" + "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" - "golang.org/x/sync/errgroup" - "gopkg.in/yaml.v2" ) type fetcherMetrics struct { @@ -278,7 +279,7 @@ type response struct { metas map[ulid.ULID]*metadata.Meta partial map[ulid.ULID]error // If metaErr > 0 it means incomplete view, so some metas, failed to be loaded. - metaErrs tsdberrors.MultiError + metaErrs errutil.MultiError noMetas float64 corruptedMetas float64 diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go index 74e971d39a75..059e8eab9d68 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "io/ioutil" + "math" "os" "path/filepath" "sort" @@ -21,11 +22,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" - terrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact/downsample" + "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/objstore" ) @@ -413,7 +414,7 @@ func (cg *Group) MinTime() int64 { cg.mtx.Lock() defer cg.mtx.Unlock() - min := int64(0) + min := int64(math.MaxInt64) for _, b := range cg.blocks { if b.MinTime < min { min = b.MinTime @@ -427,9 +428,9 @@ func (cg *Group) MaxTime() int64 { cg.mtx.Lock() defer cg.mtx.Unlock() - max := int64(0) + max := int64(math.MinInt64) for _, b := range cg.blocks { - if b.MaxTime < max { + if b.MaxTime > max { max = b.MaxTime } } @@ -515,7 +516,7 @@ func (e HaltError) Error() string { // IsHaltError returns true if the base error is a HaltError. // If a multierror is passed, any halt error will return true. func IsHaltError(err error) bool { - if multiErr, ok := errors.Cause(err).(terrors.MultiError); ok { + if multiErr, ok := errors.Cause(err).(errutil.MultiError); ok { for _, err := range multiErr { if _, ok := errors.Cause(err).(HaltError); ok { return true @@ -548,7 +549,7 @@ func (e RetryError) Error() string { // IsRetryError returns true if the base error is a RetryError. // If a multierror is passed, all errors must be retriable. func IsRetryError(err error) bool { - if multiErr, ok := errors.Cause(err).(terrors.MultiError); ok { + if multiErr, ok := errors.Cause(err).(errutil.MultiError); ok { for _, err := range multiErr { if _, ok := errors.Cause(err).(RetryError); !ok { return false @@ -962,7 +963,7 @@ func (c *BucketCompactor) Compact(ctx context.Context) (rerr error) { level.Info(c.logger).Log("msg", "start of compactions") // Send all groups found during this pass to the compaction workers. - var groupErrs terrors.MultiError + var groupErrs errutil.MultiError groupLoop: for _, g := range groups { select { @@ -984,7 +985,7 @@ func (c *BucketCompactor) Compact(ctx context.Context) (rerr error) { workCtxCancel() if len(groupErrs) > 0 { - return groupErrs + return groupErrs.Err() } if finishedAllGroups { diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go index b07fcfb47e3f..909252aee9d0 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go @@ -18,9 +18,9 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/runutil" ) @@ -73,7 +73,7 @@ func Downsample( // Remove blockDir in case of errors. defer func() { if err != nil { - var merr tsdberrors.MultiError + var merr errutil.MultiError merr.Add(err) merr.Add(os.RemoveAll(blockDir)) err = merr.Err() diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/streamed_block_writer.go b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/streamed_block_writer.go index 8a7a3f4f4875..fbe49eae4df4 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/streamed_block_writer.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/streamed_block_writer.go @@ -15,11 +15,11 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunks" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/runutil" ) @@ -61,7 +61,7 @@ func NewStreamedBlockWriter( // We should close any opened Closer up to an error. defer func() { if err != nil { - var merr tsdberrors.MultiError + var merr errutil.MultiError merr.Add(err) for _, cl := range closers { merr.Add(cl.Close()) @@ -143,7 +143,7 @@ func (w *streamedBlockWriter) Close() error { } w.finalized = true - merr := tsdberrors.MultiError{} + merr := errutil.MultiError{} if w.ignoreFinalize { // Close open file descriptors anyway. diff --git a/vendor/github.com/thanos-io/thanos/pkg/discovery/dns/provider.go b/vendor/github.com/thanos-io/thanos/pkg/discovery/dns/provider.go index 75c3b02fd198..2d11e1cf9184 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/discovery/dns/provider.go +++ b/vendor/github.com/thanos-io/thanos/pkg/discovery/dns/provider.go @@ -13,9 +13,9 @@ import ( "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/discovery/dns/miekgdns" + "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/extprom" ) @@ -111,7 +111,7 @@ func GetQTypeName(addr string) (qtype string, name string) { // defaultPort is used for non-SRV records when a port is not supplied. func (p *Provider) Resolve(ctx context.Context, addrs []string) error { resolvedAddrs := map[string][]string{} - errs := tsdberrors.MultiError{} + errs := errutil.MultiError{} for _, addr := range addrs { var resolved []string diff --git a/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go.go b/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go.go new file mode 100644 index 000000000000..e51bf4554e9d --- /dev/null +++ b/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go.go @@ -0,0 +1,51 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package errutil + +import ( + "bytes" + "fmt" +) + +// The MultiError type implements the error interface, and contains the +// Errors used to construct it. +type MultiError []error + +// Returns a concatenated string of the contained errors. +func (es MultiError) Error() string { + var buf bytes.Buffer + + if len(es) > 1 { + fmt.Fprintf(&buf, "%d errors: ", len(es)) + } + + for i, err := range es { + if i != 0 { + buf.WriteString("; ") + } + buf.WriteString(err.Error()) + } + + return buf.String() +} + +// Add adds the error to the error list if it is not nil. +func (es *MultiError) Add(err error) { + if err == nil { + return + } + if merr, ok := err.(MultiError); ok { + *es = append(*es, merr...) + } else { + *es = append(*es, err) + } +} + +// Err returns the error list as an error or nil if it is empty. +func (es MultiError) Err() error { + if len(es) == 0 { + return nil + } + return es +} diff --git a/vendor/github.com/thanos-io/thanos/pkg/objstore/azure/helpers.go b/vendor/github.com/thanos-io/thanos/pkg/objstore/azure/helpers.go index a372878ac950..9705e299cd2e 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/objstore/azure/helpers.go +++ b/vendor/github.com/thanos-io/thanos/pkg/objstore/azure/helpers.go @@ -19,6 +19,17 @@ const DirDelim = "/" var errorCodeRegex = regexp.MustCompile(`X-Ms-Error-Code:\D*\[(\w+)\]`) +func init() { + // Disable `ForceLog` in Azure storage module + // As the time of this patch, the logging function in the storage module isn't correctly + // detecting expected REST errors like 404 and so outputs them to syslog along with a stacktrace. + // https://github.com/Azure/azure-storage-blob-go/issues/214 + // + // This needs to be done at startup because the underlying variable is not thread safe. + // https://github.com/Azure/azure-pipeline-go/blob/dc95902f1d32034f8f743ccc6c3f2eb36b84da27/pipeline/core.go#L276-L283 + pipeline.SetForceLogEnabled(false) +} + func getContainerURL(ctx context.Context, conf Config) (blob.ContainerURL, error) { c, err := blob.NewSharedKeyCredential(conf.StorageAccountName, conf.StorageAccountKey) if err != nil { diff --git a/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go b/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go index 923a42539c30..cf087f5ae8da 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go +++ b/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go @@ -59,16 +59,17 @@ var DefaultConfig = Config{ // Config stores the configuration for s3 bucket. type Config struct { - Bucket string `yaml:"bucket"` - Endpoint string `yaml:"endpoint"` - Region string `yaml:"region"` - AccessKey string `yaml:"access_key"` - Insecure bool `yaml:"insecure"` - SignatureV2 bool `yaml:"signature_version2"` - SecretKey string `yaml:"secret_key"` - PutUserMetadata map[string]string `yaml:"put_user_metadata"` - HTTPConfig HTTPConfig `yaml:"http_config"` - TraceConfig TraceConfig `yaml:"trace"` + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + AccessKey string `yaml:"access_key"` + Insecure bool `yaml:"insecure"` + SignatureV2 bool `yaml:"signature_version2"` + SecretKey string `yaml:"secret_key"` + PutUserMetadata map[string]string `yaml:"put_user_metadata"` + HTTPConfig HTTPConfig `yaml:"http_config"` + TraceConfig TraceConfig `yaml:"trace"` + ListObjectsVersion string `yaml:"list_objects_version"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. PartSize uint64 `yaml:"part_size"` SSEConfig SSEConfig `yaml:"sse_config"` @@ -137,6 +138,7 @@ type Bucket struct { sse encrypt.ServerSide putUserMetadata map[string]string partSize uint64 + listObjectsV1 bool } // parseConfig unmarshals a buffer into a Config with default HTTPConfig values. @@ -246,6 +248,10 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B client.TraceOn(logWriter) } + if config.ListObjectsVersion != "" && config.ListObjectsVersion != "v1" && config.ListObjectsVersion != "v2" { + return nil, errors.Errorf("Initialize s3 client list objects version: Unsupported version %q was provided. Supported values are v1, v2", config.ListObjectsVersion) + } + bkt := &Bucket{ logger: logger, name: config.Bucket, @@ -253,6 +259,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B sse: sse, putUserMetadata: config.PutUserMetadata, partSize: config.PartSize, + listObjectsV1: config.ListObjectsVersion == "v1", } return bkt, nil } @@ -309,6 +316,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) err opts := minio.ListObjectsOptions{ Prefix: dir, Recursive: false, + UseV1: b.listObjectsV1, } for object := range b.client.ListObjects(ctx, b.name, opts) { diff --git a/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go b/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go index 5b3e85940bfe..3f817a62ee4a 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go +++ b/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go @@ -59,7 +59,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" - tsdberrors "github.com/prometheus/prometheus/tsdb/errors" + + "github.com/thanos-io/thanos/pkg/errutil" ) // Repeat executes f every interval seconds until stopc is closed or f returns an error. @@ -136,7 +137,7 @@ func ExhaustCloseWithLogOnErr(logger log.Logger, r io.ReadCloser, format string, // CloseWithErrCapture runs function and on error return error by argument including the given error (usually // from caller function). func CloseWithErrCapture(err *error, closer io.Closer, format string, a ...interface{}) { - merr := tsdberrors.MultiError{} + merr := errutil.MultiError{} merr.Add(*err) merr.Add(errors.Wrapf(closer.Close(), format, a...)) @@ -151,7 +152,7 @@ func ExhaustCloseWithErrCapture(err *error, r io.ReadCloser, format string, a .. CloseWithErrCapture(err, r, format, a...) // Prepend the io.Copy error. - merr := tsdberrors.MultiError{} + merr := errutil.MultiError{} merr.Add(copyErr) merr.Add(*err) diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go index 2c162c4885c5..6ec95eed57a1 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go @@ -707,24 +707,14 @@ func blockSeries( if err := indexr.LoadedSeries(id, &lset, &chks); err != nil { return nil, nil, errors.Wrap(err, "read series") } - s := seriesEntry{ - lset: make(labels.Labels, 0, len(lset)+len(extLset)), - refs: make([]uint64, 0, len(chks)), - chks: make([]storepb.AggrChunk, 0, len(chks)), - } - for _, l := range lset { - // Skip if the external labels of the block overrule the series' label. - // NOTE(fabxc): maybe move it to a prefixed version to still ensure uniqueness of series? - if extLset[l.Name] != "" { - continue - } - s.lset = append(s.lset, l) - } - for ln, lv := range extLset { - s.lset = append(s.lset, labels.Label{Name: ln, Value: lv}) + s := seriesEntry{lset: make(labels.Labels, 0, len(lset)+len(extLset))} + if !req.SkipChunks { + s.refs = make([]uint64, 0, len(chks)) + s.chks = make([]storepb.AggrChunk, 0, len(chks)) } - sort.Sort(s.lset) + // hasValidChunk is used to check whether there is at least one chunk in the required time range. + var hasValidChunk bool for _, meta := range chks { if meta.MaxTime < req.MinTime { continue @@ -733,6 +723,12 @@ func blockSeries( break } + // Fast path for no chunks series. + if req.SkipChunks { + hasValidChunk = true + break + } + if err := chunkr.addPreload(meta.Ref); err != nil { return nil, nil, errors.Wrap(err, "add chunk preload") } @@ -742,15 +738,37 @@ func blockSeries( }) s.refs = append(s.refs, meta.Ref) } + + // Reserve chunksLimiter if we save chunks. if len(s.chks) > 0 { + hasValidChunk = true if err := chunksLimiter.Reserve(uint64(len(s.chks))); err != nil { return nil, nil, errors.Wrap(err, "exceeded chunks limit") } + } + + if hasValidChunk { + for _, l := range lset { + // Skip if the external labels of the block overrule the series' label. + // NOTE(fabxc): maybe move it to a prefixed version to still ensure uniqueness of series? + if extLset[l.Name] != "" { + continue + } + s.lset = append(s.lset, l) + } + for ln, lv := range extLset { + s.lset = append(s.lset, labels.Label{Name: ln, Value: lv}) + } + sort.Sort(s.lset) res = append(res, s) } } + if req.SkipChunks { + return newBucketSeriesSet(res), indexr.stats, nil + } + // Preload all chunks that were marked in the previous stage. if err := chunkr.preload(); err != nil { return nil, nil, errors.Wrap(err, "preload chunks") @@ -920,13 +938,16 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie resHints.AddQueriedBlock(b.meta.ULID) } + var chunkr *bucketChunkReader // We must keep the readers open until all their data has been sent. indexr := b.indexReader(gctx) - chunkr := b.chunkReader(gctx) + if !req.SkipChunks { + chunkr = b.chunkReader(gctx) + defer runutil.CloseWithLogOnErr(s.logger, chunkr, "series block") + } // Defer all closes to the end of Series method. defer runutil.CloseWithLogOnErr(s.logger, indexr, "series block") - defer runutil.CloseWithLogOnErr(s.logger, chunkr, "series block") g.Go(func() error { part, pstats, err := blockSeries( diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go b/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go index 0301304ea78c..902611e6587e 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/multitsdb.go @@ -15,14 +15,14 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/thanos-io/thanos/pkg/runutil" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/errutil" + "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" @@ -168,7 +168,7 @@ func (s *tenantSeriesSetServer) Delegate(closer io.Closer) { } func (s *tenantSeriesSetServer) Close() error { - var merr tsdb_errors.MultiError + var merr errutil.MultiError for _, c := range s.closers { merr.Add(c.Close()) } diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/testutil/series.go b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/testutil/series.go index 213b6335bf9b..9823f12884ea 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/storepb/testutil/series.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/storepb/testutil/series.go @@ -235,10 +235,14 @@ func TestServerSeries(t testutil.TB, store storepb.StoreServer, cases ...*Series if len(c.ExpectedSeries) > 4 { for j := range c.ExpectedSeries { testutil.Equals(t, c.ExpectedSeries[j].Labels, srv.SeriesSet[j].Labels, "%v series chunks mismatch", j) - if len(c.ExpectedSeries[j].Chunks) > 20 { - testutil.Equals(t, len(c.ExpectedSeries[j].Chunks), len(srv.SeriesSet[j].Chunks), "%v series chunks number mismatch", j) + + // Check chunks when it is not a skip chunk query + if !c.Req.SkipChunks { + if len(c.ExpectedSeries[j].Chunks) > 20 { + testutil.Equals(t, len(c.ExpectedSeries[j].Chunks), len(srv.SeriesSet[j].Chunks), "%v series chunks number mismatch", j) + } + testutil.Equals(t, c.ExpectedSeries[j].Chunks, srv.SeriesSet[j].Chunks, "%v series chunks mismatch", j) } - testutil.Equals(t, c.ExpectedSeries[j].Chunks, srv.SeriesSet[j].Chunks, "%v series chunks mismatch", j) } } else { testutil.Equals(t, c.ExpectedSeries, srv.SeriesSet) diff --git a/vendor/golang.org/x/tools/internal/gocommand/invoke.go b/vendor/golang.org/x/tools/internal/gocommand/invoke.go index f516e17623d6..b5c061b0125d 100644 --- a/vendor/golang.org/x/tools/internal/gocommand/invoke.go +++ b/vendor/golang.org/x/tools/internal/gocommand/invoke.go @@ -130,6 +130,8 @@ type Invocation struct { Verb string Args []string BuildFlags []string + ModFlag string + ModFile string Env []string WorkingDir string Logf func(format string, args ...interface{}) @@ -158,17 +160,35 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { } goArgs := []string{i.Verb} + + appendModFile := func() { + if i.ModFile != "" { + goArgs = append(goArgs, "-modfile="+i.ModFile) + } + } + appendModFlag := func() { + if i.ModFlag != "" { + goArgs = append(goArgs, "-mod="+i.ModFlag) + } + } + switch i.Verb { + case "env", "version": + goArgs = append(goArgs, i.Args...) case "mod": - // mod needs the sub-verb before build flags. + // mod needs the sub-verb before flags. goArgs = append(goArgs, i.Args[0]) - goArgs = append(goArgs, i.BuildFlags...) + appendModFile() goArgs = append(goArgs, i.Args[1:]...) - case "env": - // env doesn't take build flags. + case "get": + goArgs = append(goArgs, i.BuildFlags...) + appendModFile() goArgs = append(goArgs, i.Args...) - default: + + default: // notably list and build. goArgs = append(goArgs, i.BuildFlags...) + appendModFile() + appendModFlag() goArgs = append(goArgs, i.Args...) } cmd := exec.Command("go", goArgs...) diff --git a/vendor/golang.org/x/tools/internal/imports/fix.go b/vendor/golang.org/x/tools/internal/imports/fix.go index 675d16c873b4..d859617b7745 100644 --- a/vendor/golang.org/x/tools/internal/imports/fix.go +++ b/vendor/golang.org/x/tools/internal/imports/fix.go @@ -83,7 +83,7 @@ type ImportFix struct { IdentName string // FixType is the type of fix this is (AddImport, DeleteImport, SetImportName). FixType ImportFixType - Relevance int // see pkg + Relevance float64 // see pkg } // An ImportInfo represents a single import statement. @@ -592,9 +592,9 @@ func getFixes(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv return fixes, nil } -// Highest relevance, used for the standard library. Chosen arbitrarily to -// match pre-existing gopls code. -const MaxRelevance = 7 +// MaxRelevance is the highest relevance, used for the standard library. +// Chosen arbitrarily to match pre-existing gopls code. +const MaxRelevance = 7.0 // getCandidatePkgs works with the passed callback to find all acceptable packages. // It deduplicates by import path, and uses a cached stdlib rather than reading @@ -607,6 +607,10 @@ func getCandidatePkgs(ctx context.Context, wrappedCallback *scanCallback, filena if err != nil { return err } + + var mu sync.Mutex // to guard asynchronous access to dupCheck + dupCheck := map[string]struct{}{} + // Start off with the standard library. for importPath, exports := range stdlib { p := &pkg{ @@ -615,14 +619,12 @@ func getCandidatePkgs(ctx context.Context, wrappedCallback *scanCallback, filena packageName: path.Base(importPath), relevance: MaxRelevance, } + dupCheck[importPath] = struct{}{} if notSelf(p) && wrappedCallback.dirFound(p) && wrappedCallback.packageNameLoaded(p) { wrappedCallback.exportsLoaded(p, exports) } } - var mu sync.Mutex - dupCheck := map[string]struct{}{} - scanFilter := &scanCallback{ rootFound: func(root gopathwalk.Root) bool { // Exclude goroot results -- getting them is relatively expensive, not cached, @@ -658,8 +660,8 @@ func getCandidatePkgs(ctx context.Context, wrappedCallback *scanCallback, filena return resolver.scan(ctx, scanFilter) } -func ScoreImportPaths(ctx context.Context, env *ProcessEnv, paths []string) (map[string]int, error) { - result := make(map[string]int) +func ScoreImportPaths(ctx context.Context, env *ProcessEnv, paths []string) (map[string]float64, error) { + result := make(map[string]float64) resolver, err := env.GetResolver() if err != nil { return nil, err @@ -802,6 +804,8 @@ type ProcessEnv struct { GocmdRunner *gocommand.Runner BuildFlags []string + ModFlag string + ModFile string // Env overrides the OS environment, and can be used to specify // GOPROXY, GO111MODULE, etc. PATH cannot be set here, because @@ -995,7 +999,7 @@ type Resolver interface { // loadExports may be called concurrently. loadExports(ctx context.Context, pkg *pkg, includeTest bool) (string, []string, error) // scoreImportPath returns the relevance for an import path. - scoreImportPath(ctx context.Context, path string) int + scoreImportPath(ctx context.Context, path string) float64 ClearForNewScan() } @@ -1260,10 +1264,10 @@ func packageDirToName(dir string) (packageName string, err error) { } type pkg struct { - dir string // absolute file path to pkg directory ("/usr/lib/go/src/net/http") - importPathShort string // vendorless import path ("net/http", "a/b") - packageName string // package name loaded from source if requested - relevance int // a weakly-defined score of how relevant a package is. 0 is most relevant. + dir string // absolute file path to pkg directory ("/usr/lib/go/src/net/http") + importPathShort string // vendorless import path ("net/http", "a/b") + packageName string // package name loaded from source if requested + relevance float64 // a weakly-defined score of how relevant a package is. 0 is most relevant. } type pkgDistance struct { @@ -1389,7 +1393,7 @@ func (r *gopathResolver) scan(ctx context.Context, callback *scanCallback) error return nil } -func (r *gopathResolver) scoreImportPath(ctx context.Context, path string) int { +func (r *gopathResolver) scoreImportPath(ctx context.Context, path string) float64 { if _, ok := stdlib[path]; ok { return MaxRelevance } diff --git a/vendor/golang.org/x/tools/internal/imports/mod.go b/vendor/golang.org/x/tools/internal/imports/mod.go index 94880d616041..8a83613c5728 100644 --- a/vendor/golang.org/x/tools/internal/imports/mod.go +++ b/vendor/golang.org/x/tools/internal/imports/mod.go @@ -59,6 +59,8 @@ func (r *ModuleResolver) init() error { } inv := gocommand.Invocation{ BuildFlags: r.env.BuildFlags, + ModFlag: r.env.ModFlag, + ModFile: r.env.ModFile, Env: r.env.env(), Logf: r.env.Logf, WorkingDir: r.env.WorkingDir, @@ -487,7 +489,7 @@ func (r *ModuleResolver) scan(ctx context.Context, callback *scanCallback) error return nil } -func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) int { +func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) float64 { if _, ok := stdlib[path]; ok { return MaxRelevance } @@ -495,17 +497,31 @@ func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) int { return modRelevance(mod) } -func modRelevance(mod *gocommand.ModuleJSON) int { +func modRelevance(mod *gocommand.ModuleJSON) float64 { + var relevance float64 switch { case mod == nil: // out of scope return MaxRelevance - 4 case mod.Indirect: - return MaxRelevance - 3 + relevance = MaxRelevance - 3 case !mod.Main: - return MaxRelevance - 2 + relevance = MaxRelevance - 2 default: - return MaxRelevance - 1 // main module ties with stdlib + relevance = MaxRelevance - 1 // main module ties with stdlib } + + _, versionString, ok := module.SplitPathVersion(mod.Path) + if ok { + index := strings.Index(versionString, "v") + if index == -1 { + return relevance + } + if versionNumber, err := strconv.ParseFloat(versionString[index+1:], 64); err == nil { + relevance += versionNumber / 1000 + } + } + + return relevance } // canonicalize gets the result of canonicalizing the packages using the results diff --git a/vendor/modules.txt b/vendor/modules.txt index 108c41af8aee..823ab45fceba 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -48,6 +48,7 @@ github.com/Masterminds/squirrel github.com/Microsoft/go-winio github.com/Microsoft/go-winio/pkg/guid # github.com/NYTimes/gziphandler v1.1.1 +## explicit github.com/NYTimes/gziphandler # github.com/PuerkitoBio/purell v1.1.1 github.com/PuerkitoBio/purell @@ -159,7 +160,7 @@ github.com/coreos/go-systemd/journal github.com/coreos/go-systemd/sdjournal # github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/coreos/pkg/capnslog -# github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf +# github.com/cortexproject/cortex v1.5.1-0.20201110164020-35e698bb56d6 ## explicit github.com/cortexproject/cortex/pkg/alertmanager github.com/cortexproject/cortex/pkg/alertmanager/alerts @@ -194,6 +195,12 @@ github.com/cortexproject/cortex/pkg/configs/userconfig github.com/cortexproject/cortex/pkg/cortex github.com/cortexproject/cortex/pkg/distributor github.com/cortexproject/cortex/pkg/flusher +github.com/cortexproject/cortex/pkg/frontend +github.com/cortexproject/cortex/pkg/frontend/transport +github.com/cortexproject/cortex/pkg/frontend/v1 +github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb +github.com/cortexproject/cortex/pkg/frontend/v2 +github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb github.com/cortexproject/cortex/pkg/ingester github.com/cortexproject/cortex/pkg/ingester/client github.com/cortexproject/cortex/pkg/ingester/index @@ -202,11 +209,11 @@ github.com/cortexproject/cortex/pkg/querier github.com/cortexproject/cortex/pkg/querier/astmapper github.com/cortexproject/cortex/pkg/querier/batch github.com/cortexproject/cortex/pkg/querier/chunkstore -github.com/cortexproject/cortex/pkg/querier/frontend github.com/cortexproject/cortex/pkg/querier/iterators github.com/cortexproject/cortex/pkg/querier/lazyquery github.com/cortexproject/cortex/pkg/querier/queryrange github.com/cortexproject/cortex/pkg/querier/series +github.com/cortexproject/cortex/pkg/querier/worker github.com/cortexproject/cortex/pkg/ring github.com/cortexproject/cortex/pkg/ring/client github.com/cortexproject/cortex/pkg/ring/kv @@ -218,10 +225,14 @@ github.com/cortexproject/cortex/pkg/ruler github.com/cortexproject/cortex/pkg/ruler/rules github.com/cortexproject/cortex/pkg/ruler/rules/local github.com/cortexproject/cortex/pkg/ruler/rules/objectclient +github.com/cortexproject/cortex/pkg/scheduler +github.com/cortexproject/cortex/pkg/scheduler/queue +github.com/cortexproject/cortex/pkg/scheduler/schedulerpb github.com/cortexproject/cortex/pkg/storage/backend/azure github.com/cortexproject/cortex/pkg/storage/backend/filesystem github.com/cortexproject/cortex/pkg/storage/backend/gcs github.com/cortexproject/cortex/pkg/storage/backend/s3 +github.com/cortexproject/cortex/pkg/storage/backend/swift github.com/cortexproject/cortex/pkg/storage/tsdb github.com/cortexproject/cortex/pkg/storegateway github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb @@ -234,6 +245,7 @@ github.com/cortexproject/cortex/pkg/util/grpc github.com/cortexproject/cortex/pkg/util/grpc/encoding/snappy github.com/cortexproject/cortex/pkg/util/grpc/healthcheck github.com/cortexproject/cortex/pkg/util/grpcclient +github.com/cortexproject/cortex/pkg/util/grpcutil github.com/cortexproject/cortex/pkg/util/limiter github.com/cortexproject/cortex/pkg/util/middleware github.com/cortexproject/cortex/pkg/util/modules @@ -727,7 +739,7 @@ github.com/prometheus/node_exporter/https github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643 +# github.com/prometheus/prometheus v1.8.2-0.20201029103703-63be30dceed9 ## explicit github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -824,7 +836,7 @@ github.com/stretchr/objx github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/thanos-io/thanos v0.13.1-0.20201019130456-f41940581d9a +# github.com/thanos-io/thanos v0.13.1-0.20201030101306-47f9a225cc52 github.com/thanos-io/thanos/pkg/block github.com/thanos-io/thanos/pkg/block/indexheader github.com/thanos-io/thanos/pkg/block/metadata @@ -836,6 +848,7 @@ github.com/thanos-io/thanos/pkg/component github.com/thanos-io/thanos/pkg/discovery/cache github.com/thanos-io/thanos/pkg/discovery/dns github.com/thanos-io/thanos/pkg/discovery/dns/miekgdns +github.com/thanos-io/thanos/pkg/errutil github.com/thanos-io/thanos/pkg/extprom github.com/thanos-io/thanos/pkg/gate github.com/thanos-io/thanos/pkg/http @@ -1108,7 +1121,7 @@ golang.org/x/text/unicode/norm golang.org/x/text/width # golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e golang.org/x/time/rate -# golang.org/x/tools v0.0.0-20201008025239-9df69603baec +# golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca golang.org/x/tools/cmd/goimports golang.org/x/tools/go/ast/astutil golang.org/x/tools/go/gcexportdata From 84a21d6fa1b1e23bacf2b40367d78e16e09ff6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 12 Nov 2020 08:35:33 +0100 Subject: [PATCH 2/3] Ignore set scheduler address. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/loki/modules.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 5ab798d7e90c..feaeabae7275 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -166,6 +166,8 @@ func (t *Loki) initQuerier() (services.Service, error) { // NewQuerierWorker now expects Frontend (or Scheduler) address to be set. Loki only supports Frontend for now. if t.cfg.Worker.FrontendAddress != "" { + // In case someone set scheduler address, we ignore it. + t.cfg.Worker.SchedulerAddress = "" t.cfg.Worker.MaxConcurrentRequests = t.cfg.Querier.MaxConcurrent level.Debug(util.Logger).Log("msg", "initializing querier worker", "config", fmt.Sprintf("%+v", t.cfg.Worker)) worker, err = cortex_querier_worker.NewQuerierWorker(t.cfg.Worker, httpgrpc_server.NewServer(t.server.HTTPServer.Handler), util.Logger, prometheus.DefaultRegisterer) From 1ef3b78ebbc160358773dc3d76666b7341442895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 12 Nov 2020 16:15:06 +0100 Subject: [PATCH 3/3] Make linter happy. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/lokifrontend/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lokifrontend/config.go b/pkg/lokifrontend/config.go index ba0826c3d4f3..31736db11293 100644 --- a/pkg/lokifrontend/config.go +++ b/pkg/lokifrontend/config.go @@ -4,7 +4,7 @@ import ( "flag" "github.com/cortexproject/cortex/pkg/frontend/transport" - "github.com/cortexproject/cortex/pkg/frontend/v1" + v1 "github.com/cortexproject/cortex/pkg/frontend/v1" ) type Config struct {