Add Store sharding and Receive hashrings

CHANGELOG.md

@@ -19,7 +19,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Added
 
--
+- [#170](https://github.com/thanos-io/kube-thanos/pull/170) Add Store shard and Receive hashring helpers
 
 ### Fixed
 

all.jsonnet

@@ -112,11 +112,9 @@ local q = t.query(commonConfig {
   logLevel: 'debug',
 });
 
-local finalRu = ru {
-  config+:: {
-    queriers: ['dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [q.service.metadata.name, q.service.metadata.namespace]],
-  },
-};
+local finalRu = t.rule(ru.config {
+  queriers: ['dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [q.service.metadata.name, q.service.metadata.namespace]],
+});
 
 local qf = t.queryFrontend(commonConfig {
   replicas: 1,
@@ -149,10 +147,72 @@ local qf = t.queryFrontend(commonConfig {
   },
 });
 
+local rcvs = t.receiveHashrings(commonConfig {
+  hashrings: [
+    {
+      hashring: 'default',
+      tenants: [],
+    },
+    {
+      hashring: 'region-1',
+      tenants: [],
+    },
+  ],
+  replicas: 1,
+  replicationFactor: 1,
+  serviceMonitor: true,
+  hashringConfigMapName: 'hashring',
+});
+
+local strs = t.storeShards(commonConfig {
+  shards: 3,
+  replicas: 1,
+  serviceMonitor: true,
+  bucketCache: {
+    type: 'memcached',
+    config+: {
+      // NOTICE: <MEMCACHED_SERCIVE> is a placeholder to generate examples.
+      // List of memcached addresses, that will get resolved with the DNS service discovery provider.
+      // For DNS service discovery reference https://thanos.io/service-discovery.md/#dns-service-discovery
+      addresses: ['dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.%s.svc.cluster.local' % commonConfig.namespace],
+    },
+  },
+  indexCache: {
+    type: 'memcached',
+    config+: {
+      // NOTICE: <MEMCACHED_SERCIVE> is a placeholder to generate examples.
+      // List of memcached addresses, that will get resolved with the DNS service discovery provider.
+      // For DNS service discovery reference https://thanos.io/service-discovery.md/#dns-service-discovery
+      addresses: ['dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.%s.svc.cluster.local' % commonConfig.namespace],
+    },
+  },
+});
+
+local finalQ = t.query(q.config {
+  stores: [
+    'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [service.metadata.name, service.metadata.namespace]
+    for service in [re.service, ru.service, s.service] +
+                   [rcvs[hashring].service for hashring in std.objectFields(rcvs)] +
+                   [strs[shard].service for shard in std.objectFields(strs)]
+  ],
+});
+
 { ['thanos-bucket-' + name]: b[name] for name in std.objectFields(b) } +
 { ['thanos-compact-' + name]: c[name] for name in std.objectFields(c) } +
 { ['thanos-receive-' + name]: re[name] for name in std.objectFields(re) } +
 { ['thanos-rule-' + name]: finalRu[name] for name in std.objectFields(finalRu) } +
 { ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } +
-{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) } +
-{ ['thanos-query-frontend-' + name]: qf[name] for name in std.objectFields(qf) }
+{ ['thanos-query-' + name]: finalQ[name] for name in std.objectFields(finalQ) } +
+{ ['thanos-query-frontend-' + name]: qf[name] for name in std.objectFields(qf) } +
+{
+  ['thanos-receive-' + hashring + '-' + name]: rcvs[hashring][name]
+  for hashring in std.objectFields(rcvs)
+  for name in std.objectFields(rcvs[hashring])
+  if rcvs[hashring][name] != null
+} +
+{
+  ['store-' + shard + '-' + name]: strs[shard][name]
+  for shard in std.objectFields(strs)
+  for name in std.objectFields(strs[shard])
+  if strs[shard][name] != null
+}

examples/all/manifests/store-shard0-service.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-0
+    app.kubernetes.io/name: thanos-store
+    app.kubernetes.io/version: v0.16.0
+    store.observatorium.io/shard: shard-0
+  name: thanos-store-0
+  namespace: thanos
+spec:
+  clusterIP: None
+  ports:
+  - name: grpc
+    port: 10901
+    targetPort: 10901
+  - name: http
+    port: 10902
+    targetPort: 10902
+  selector:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-0
+    app.kubernetes.io/name: thanos-store
+    store.observatorium.io/shard: shard-0

examples/all/manifests/store-shard0-serviceMonitor.yaml

@@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-0
+    app.kubernetes.io/name: thanos-store
+    app.kubernetes.io/version: v0.16.0
+    store.observatorium.io/shard: shard-0
+  name: thanos-store-0
+  namespace: thanos
+spec:
+  endpoints:
+  - port: http
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: object-store-gateway
+      app.kubernetes.io/instance: thanos-store-0
+      app.kubernetes.io/name: thanos-store
+      store.observatorium.io/shard: shard-0

examples/all/manifests/store-shard0-statefulSet.yaml

@@ -0,0 +1,162 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-0
+    app.kubernetes.io/name: thanos-store
+    app.kubernetes.io/version: v0.16.0
+    store.observatorium.io/shard: shard-0
+  name: thanos-store-0
+  namespace: thanos
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: object-store-gateway
+      app.kubernetes.io/instance: thanos-store-0
+      app.kubernetes.io/name: thanos-store
+      store.observatorium.io/shard: shard-0
+  serviceName: thanos-store-0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: object-store-gateway
+        app.kubernetes.io/instance: thanos-store-0
+        app.kubernetes.io/name: thanos-store
+        app.kubernetes.io/version: v0.16.0
+        store.observatorium.io/shard: shard-0
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-store
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-store-0
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - store
+        - --log.level=info
+        - --data-dir=/var/thanos/store
+        - --grpc-address=0.0.0.0:10901
+        - --http-address=0.0.0.0:10902
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --ignore-deletion-marks-delay=24h
+        - --experimental.enable-index-cache-postings-compression
+        - |-
+          --index-cache.config="config":
+            "addresses":
+            - "dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.thanos.svc.cluster.local"
+            "dns_provider_update_interval": "10s"
+            "max_async_buffer_size": 10000
+            "max_async_concurrency": 20
+            "max_get_multi_batch_size": 0
+            "max_get_multi_concurrency": 100
+            "max_idle_connections": 100
+            "max_item_size": "1MiB"
+            "timeout": "500ms"
+          "type": "memcached"
+        - |-
+          --store.caching-bucket.config="blocks_iter_ttl": "5m"
+          "chunk_object_attrs_ttl": "24h"
+          "chunk_subrange_size": 16000
+          "chunk_subrange_ttl": "24h"
+          "config":
+            "addresses":
+            - "dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.thanos.svc.cluster.local"
+            "dns_provider_update_interval": "10s"
+            "max_async_buffer_size": 10000
+            "max_async_concurrency": 20
+            "max_get_multi_batch_size": 0
+            "max_get_multi_concurrency": 100
+            "max_idle_connections": 100
+            "max_item_size": "1MiB"
+            "timeout": "500ms"
+          "max_chunks_get_range_requests": 3
+          "metafile_content_ttl": "24h"
+          "metafile_doesnt_exist_ttl": "15m"
+          "metafile_exists_ttl": "2h"
+          "metafile_max_size": "1MiB"
+          "type": "memcached"
+        - |-
+          --tracing.config="config":
+            "sampler_param": 2
+            "sampler_type": "ratelimiting"
+            "service_name": "thanos-store"
+          "type": "JAEGER"
+        - |
+          --selector.relabel-config=
+            - action: hashmod
+              source_labels: ["__block_id"]
+              target_label: shard
+              modulus: 3
+            - action: keep
+              source_labels: ["shard"]
+              regex: 0
+        env:
+        - name: OBJSTORE_CONFIG
+          valueFrom:
+            secretKeyRef:
+              key: thanos.yaml
+              name: thanos-objectstorage
+        image: quay.io/thanos/thanos:v0.16.0
+        livenessProbe:
+          failureThreshold: 8
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
+        name: thanos-store
+        ports:
+        - containerPort: 10901
+          name: grpc
+        - containerPort: 10902
+          name: http
+        readinessProbe:
+          failureThreshold: 20
+          httpGet:
+            path: /-/ready
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 5
+        resources:
+          limits:
+            cpu: 0.42
+            memory: 420Mi
+          requests:
+            cpu: 0.123
+            memory: 123Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/thanos/store
+          name: data
+          readOnly: false
+      terminationGracePeriodSeconds: 120
+      volumes: []
+  volumeClaimTemplates:
+  - metadata:
+      labels:
+        app.kubernetes.io/component: object-store-gateway
+        app.kubernetes.io/instance: thanos-store-0
+        app.kubernetes.io/name: thanos-store
+        store.observatorium.io/shard: shard-0
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi

examples/all/manifests/store-shard1-service.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-1
+    app.kubernetes.io/name: thanos-store
+    app.kubernetes.io/version: v0.16.0
+    store.observatorium.io/shard: shard-1
+  name: thanos-store-1
+  namespace: thanos
+spec:
+  clusterIP: None
+  ports:
+  - name: grpc
+    port: 10901
+    targetPort: 10901
+  - name: http
+    port: 10902
+    targetPort: 10902
+  selector:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-1
+    app.kubernetes.io/name: thanos-store
+    store.observatorium.io/shard: shard-1

examples/all/manifests/store-shard1-serviceMonitor.yaml

@@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: object-store-gateway
+    app.kubernetes.io/instance: thanos-store-1
+    app.kubernetes.io/name: thanos-store
+    app.kubernetes.io/version: v0.16.0
+    store.observatorium.io/shard: shard-1
+  name: thanos-store-1
+  namespace: thanos
+spec:
+  endpoints:
+  - port: http
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: object-store-gateway
+      app.kubernetes.io/instance: thanos-store-1
+      app.kubernetes.io/name: thanos-store
+      store.observatorium.io/shard: shard-1