Merge pull request #414 from grafana/chore/tsdb-add-multi-concurrency

tsdb: Add multi concurrency and max idle connections store gateway params

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## master / unreleased
 
+* [CHANGE] Store gateway: set `-blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency`,
+  `-blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency`,
+  `-blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency`,
+  `-blocks-storage.bucket-store.index-cache.memcached.max-idle-connections`,
+  `-blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections`,
+  `-blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections` to 100 #414
 * [CHANGE] Update grafana-builder dependency: use $__rate_interval in qpsPanel and latencyPanel. #372
 * [CHANGE] `namespace` template variable in dashboards now only selects namespaces for selected clusters. #311
 * [CHANGE] Alertmanager: mounted overrides configmap to alertmanager too. #315

cortex/tsdb.libsonnet

@@ -223,6 +223,20 @@
       'blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout': '60m',
 
       'blocks-storage.bucket-store.max-chunk-pool-bytes': 12 * 1024 * 1024 * 1024,
+
+      // We should keep a number of idle connections equal to the max "get" concurrency,
+      // in order to avoid re-opening connections continuously (this would be slower
+      // and fill up the conntrack table too).
+      //
+      // The downside of this approach is that we'll end up with an higher number of
+      // active connections to memcached, so we have to make sure connections limit
+      // set in memcached is high enough.
+      'blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency': 100,
+      'blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency': 100,
+      'blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency': 100,
+      'blocks-storage.bucket-store.index-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency'],
+      'blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency'],
+      'blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency'],
     } +
     $.blocks_chunks_caching_config +
     $.blocks_metadata_caching_config +