Zone aware ingesters (#1936)

* add zone aware replication for ingesters

* jsonnet fmt

* add check for multizone_zone_ingester_enabled in example

* add readme file for jsonnet

CHANGELOG.md

@@ -1,4 +1,12 @@
 ## main / unreleased
+* [ENHANCEMENT] Add zone awareness replication for ingesters. [#1936](https://github.com/grafana/tempo/pull/1936) (@manohar-koukuntla)
+```
+# use the following fields in _config field of jsonnet config, to enable zone aware ingester
+    multi_zone_ingester_enabled: false,
+    multi_zone_ingester_migration_enabled: false,
+    multi_zone_ingester_replicas: 0,
+    multi_zone_ingester_max_unavailable: 25,
+```
 * [BUGFIX] Stop distributors on Otel receiver fatal error[#1887](https://github.com/grafana/tempo/pull/1887) (@rdooley)
 * [CHANGE] **BREAKING CHANGE** Use snake case on Azure Storage config [#1879](https://github.com/grafana/tempo/issues/1879) (@faustodavid)
 Example of using snake case on Azure Storage config:

operations/jsonnet-compiled/Deployment-compactor.yaml

@@ -25,9 +25,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=compactor
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=compactor
         env:
         - name: GOMEMLIMIT
           value: 5GiB

operations/jsonnet-compiled/Deployment-distributor.yaml

@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=distributor
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=distributor
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: distributor

operations/jsonnet-compiled/Deployment-metrics-generator.yaml

@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=metrics-generator
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=metrics-generator
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: metrics-generator

operations/jsonnet-compiled/Deployment-querier.yaml

@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=querier
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=querier
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: querier

operations/jsonnet-compiled/Deployment-query-frontend.yaml

@@ -25,9 +25,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=query-frontend
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=query-frontend
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: query-frontend

operations/jsonnet-compiled/StatefulSet-ingester.yaml

@@ -30,9 +30,9 @@ spec:
             topologyKey: kubernetes.io/hostname
       containers:
       - args:
-        - -target=ingester
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=ingester
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: ingester
@@ -59,6 +59,7 @@ spec:
           name: ingester-data
         - mountPath: /overrides
           name: overrides
+      terminationGracePeriodSeconds: 1200
       volumes:
       - configMap:
           name: tempo-ingester

operations/jsonnet-compiled/util/example/main.jsonnet

@@ -65,7 +65,7 @@ tempo {
   },
 
   local statefulSet = $.apps.v1.statefulSet,
-  tempo_ingester_statefulset+:
-    statefulSet.mixin.spec.withPodManagementPolicy('Parallel'),
+  tempo_ingester_statefulset:
+    if !$._config.multi_zone_ingester_enabled then super.tempo_ingester_statefulset + statefulSet.mixin.spec.withPodManagementPolicy('Parallel') else null,
 
 }

operations/jsonnet-compiled/util/jsonnetfile.lock.json

@@ -8,7 +8,7 @@
           "subdir": "ksonnet-util"
         }
       },
-      "version": "32101ab7cf57915901a75af9c2cf2442416677f6",
+      "version": "e6a0083d9cc0f0ec79507397ce0e156d558f6efb",
       "sum": "/pkNOLhRqvQoPA0yYdUuJvpPHqhkCLauAUMD2ZHMIkE="
     },
     {
@@ -18,7 +18,7 @@
           "subdir": "memcached"
         }
       },
-      "version": "32101ab7cf57915901a75af9c2cf2442416677f6",
+      "version": "e6a0083d9cc0f0ec79507397ce0e156d558f6efb",
       "sum": "SWywAq4U0MRPMbASU0Ez8O9ArRNeoZzb75sEuReueow="
     },
     {

operations/jsonnet/microservices/Readme.md

@@ -0,0 +1,89 @@
+# Deploy Grafana Tempo with Jsonnet and Tanka
+You can use Tanka and jsonnet-bundler to generate Kubernetes YAML manifests from the jsonnet files.
+
+1. Install tanka and jb:
+
+   Follow the steps at https://tanka.dev/install. If you have go installed locally you can also use:
+
+```shell
+# make sure to be outside of GOPATH or a go.mod project
+go install github.com/grafana/tanka/cmd/tk@latest
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+```
+
+2. Set up a Jsonnet project, based on the example that follows:
+
+  * Initialize Tanka 
+  * Install Grafana Tempo and Kubernetes Jsonnet libraries 
+  * Set up an environment
+
+   ```shell
+   #!/usr/bin/env bash
+   # SPDX-License-Identifier: AGPL-3.0-only
+
+   set -e
+
+   # Initialise the Tanka.
+   mkdir jsonnet-example && cd jsonnet-example
+   tk init --k8s=1.21
+
+   # Install Tempo jsonnet.
+   jb install github.com/grafana/tempo/operations/jsonnet/microservices@main
+   jb install github.com/grafana/jsonnet-libs/memcached
+
+   # Use the provided example. In tempo repository in operations/jsonnet-compiled
+   cp operations/jsonnet-compliled/util/example/main.jsonnet environments/default/main.jsonnet
+
+   # Generate the YAML manifests.
+   export PAGER=cat
+   tk show environments/default
+   ```
+
+3. Generate the Kubernetes YAML manifests and store them in the ./manifests directory:
+
+   ```shell
+   # Generate the YAML manifests:
+   export PAGER=cat
+   tk show environments/default
+   tk export manifests environments/default
+   ```
+4. Configure the environment specification file at environments/default/spec.json.
+
+To learn about how to use Tanka and to configure the spec.json file, see Using Jsonnet: Creating a new project.
+
+5. Deploy the manifests to a Kubernetes cluster, in one of two ways:
+   * Use the tk apply command.
+
+   Tanka supports commands to show the diff and apply changes to a Kubernetes cluster:
+
+   ```shell
+   # Show the difference between your Jsonnet definition and your Kubernetes cluster:
+   tk diff environments/default
+
+   # Apply changes to your Kubernetes cluster:
+   tk apply environments/default
+   ```
+
+   * Use the kubectl apply command.
+
+   You generated the Kubernetes manifests and stored them in the ./manifests directory in the previous step.
+
+   You can run the following command to directly apply these manifests to your Kubernetes cluster:
+
+   ```shell
+   # Review the changes that will apply to your Kubernetes cluster:
+   kubectl apply --dry-run=client -k manifests/
+
+   # Apply the changes to your Kubernetes cluster:
+   kubectl apply -k manifests/
+   ```
+6. Multizone ingesters
+   To use multizone ingesters use following config fields
+   ```
+    _config+: {
+        multi_zone_ingester_enabled: false,
+        multi_zone_ingester_migration_enabled: false,
+        multi_zone_ingester_replicas: 0,
+        multi_zone_ingester_max_unavailable: 25,
+   }
+   ```

operations/jsonnet/microservices/compactor.libsonnet

@@ -12,24 +12,25 @@
   local tempo_data_volume = 'tempo-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_compactor_ports:: [containerPort.new('prom-metrics', $._config.port)],
+  tempo_compactor_args:: {
+    target: target_name,
+    'config.file': '/conf/tempo.yaml',
+    'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+  },
+
   tempo_compactor_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_compactor_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_compactor_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.compactor.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}) +
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}) +
     (if $._config.compactor.resources.limits.memory != null then container.withEnvMixin([envVar.new('GOMEMLIMIT', $._config.compactor.resources.limits.memory + 'B')]) else {}),
 
   tempo_compactor_deployment:

operations/jsonnet/microservices/config.libsonnet

@@ -3,14 +3,21 @@
     tempo: 'grafana/tempo:latest',
     tempo_query: 'grafana/tempo-query:latest',
     tempo_vulture: 'grafana/tempo-vulture:latest',
+    rollout_operator: 'grafana/rollout-operator:v0.1.1',
     memcached: 'memcached:1.6.9-alpine',
     memcachedExporter: 'prom/memcached-exporter:v0.6.0',
   },
 
   _config+:: {
     gossip_member_label: 'tempo-gossip-member',
+    // Labels that service selectors should not use
+    service_ignored_labels:: [self.gossip_member_label],
+
     variables_expansion: false,
     variables_expansion_env_mixin: null,
+    node_selector: null,
+    ingester_allow_multiple_replicas_on_same_node: false,
+
     compactor: {
       replicas: 1,
       resources: {

operations/jsonnet/microservices/distributor.libsonnet

@@ -11,24 +11,25 @@
   local tempo_config_volume = 'tempo-conf',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_distributor_ports:: [containerPort.new('prom-metrics', $._config.port)],
+  tempo_distributor_args:: {
+    target: target_name,
+    'config.file': '/conf/tempo.yaml',
+    'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+  },
+
   tempo_distributor_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_distributor_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_distributor_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.distributor.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_distributor_deployment:
     deployment.new(target_name,

operations/jsonnet/microservices/frontend.libsonnet

@@ -14,24 +14,25 @@
   local tempo_data_volume = 'tempo-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_query_frontend_ports:: [containerPort.new('prom-metrics', $._config.port)],
+  tempo_query_frontend_args:: {
+    target: target_name,
+    'config.file': '/conf/tempo.yaml',
+    'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+  },
+
   tempo_query_frontend_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_query_frontend_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_query_frontend_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.query_frontend.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_query_container::
     container.new('tempo-query', $._images.tempo_query) +

operations/jsonnet/microservices/generator.libsonnet

@@ -12,16 +12,17 @@
   local tempo_generator_wal_volume = 'metrics-generator-wal-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_metrics_generator_ports:: [containerPort.new('prom-metrics', $._config.port)],
+  tempo_metrics_generator_args:: {
+    target: target_name,
+    'config.file': '/conf/tempo.yaml',
+    'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+  },
+
   tempo_metrics_generator_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_metrics_generator_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_metrics_generator_args)) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_generator_wal_volume, $.tempo_metrics_generator_config.metrics_generator.storage.path),
@@ -32,7 +33,7 @@
     container.mixin.resources.withRequestsMixin({ 'ephemeral-storage': $._config.metrics_generator.ephemeral_storage_request_size }) +
     container.mixin.resources.withLimitsMixin({ 'ephemeral-storage': $._config.metrics_generator.ephemeral_storage_limit_size }) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_metrics_generator_deployment:
     deployment.new(