receive.json

{
   "annotations": {
      "list": [ ]
   },
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
   "hideControls": false,
   "links": [ ],
   "refresh": "10s",
   "rows": [
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of incoming requests.",
               "fill": 10,
               "id": 1,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "/1../",
                     "color": "#EAB839"
                  },
                  {
                     "alias": "/2../",
                     "color": "#37872D"
                  },
                  {
                     "alias": "/3../",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/4../",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/5../",
                     "color": "#C4162A"
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{job}} {{handler}} {{code}}",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of handled incoming requests.",
               "fill": 10,
               "id": 2,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows how long has it taken to handle incoming requests in quantiles.",
               "fill": 1,
               "id": 3,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "p99",
                     "color": "#FA6400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p90",
                     "color": "#E0B400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p50",
                     "color": "#37872D",
                     "fill": 10,
                     "fillGradient": 0
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p50 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p90 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p99 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "WRITE - Incoming Request",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 4,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{code}} - {{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate of write requests (by tenant and code)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 5,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{code}} - {{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Number of errors (by tenant and code)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 6,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, tenant) (rate(http_request_duration_seconds_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval])) / sum by (job, tenant) (http_request_duration_seconds_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Average request duration (by tenant)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "WRITE - Incoming Request (tenant focus)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 7,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Average successful HTTP request size (per tenant and code, only 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 8,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Average failed HTTP request size (per tenant and code, non 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 9,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, tenant, method) (http_inflight_requests{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{method}} - {{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Inflight requests (per tenant and method)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "HTTP requests (tenant focus)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 10,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate of series received (per tenant, only 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 11,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{code}} - {{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate of series not written (per tenant and code, non 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 12,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate of samples received (per tenant, only 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 13,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{code}} - {{tenant}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate of samples not written (per tenant and code, non 2XX)",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Series & Samples (tenant focus)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of replications to other receive nodes.",
               "fill": 1,
               "id": 14,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "all {{job}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of replications to other receive nodes.",
               "fill": 10,
               "id": 15,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "error",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "WRITE - Replication",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of forwarded requests to other receive nodes.",
               "fill": 1,
               "id": 16,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$__rate_interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "all {{job}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.",
               "fill": 10,
               "id": 17,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "error",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "WRITE - Forward Request",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of handled Unary gRPC requests from queriers.",
               "fill": 10,
               "id": 18,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "/Aborted/",
                     "color": "#EAB839"
                  },
                  {
                     "alias": "/AlreadyExists/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "/FailedPrecondition/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/Unimplemented/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/InvalidArgument/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/NotFound/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/PermissionDenied/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Unauthenticated/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Canceled/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DataLoss/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DeadlineExceeded/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Internal/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OutOfRange/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/ResourceExhausted/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unavailable/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unknown/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OK/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "error",
                     "color": "#C4162A"
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of handled requests from queriers.",
               "fill": 10,
               "id": 19,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows how long has it taken to handle requests from queriers, in quantiles.",
               "fill": 1,
               "id": 20,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "p99",
                     "color": "#FA6400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p90",
                     "color": "#E0B400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p50",
                     "color": "#37872D",
                     "fill": 10,
                     "fillGradient": 0
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p50 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p90 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p99 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "WRITE - gRPC (Unary)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of handled Unary gRPC requests from queriers.",
               "fill": 10,
               "id": 21,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "/Aborted/",
                     "color": "#EAB839"
                  },
                  {
                     "alias": "/AlreadyExists/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "/FailedPrecondition/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/Unimplemented/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/InvalidArgument/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/NotFound/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/PermissionDenied/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Unauthenticated/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Canceled/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DataLoss/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DeadlineExceeded/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Internal/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OutOfRange/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/ResourceExhausted/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unavailable/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unknown/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OK/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "error",
                     "color": "#C4162A"
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of handled requests from queriers.",
               "fill": 10,
               "id": 22,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows how long has it taken to handle requests from queriers, in quantiles.",
               "fill": 1,
               "id": 23,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "p99",
                     "color": "#FA6400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p90",
                     "color": "#E0B400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p50",
                     "color": "#37872D",
                     "fill": 10,
                     "fillGradient": 0
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p50 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p90 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p99 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "READ - gRPC (Unary)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows rate of handled Streamed gRPC requests from queriers.",
               "fill": 10,
               "id": 24,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "/Aborted/",
                     "color": "#EAB839"
                  },
                  {
                     "alias": "/AlreadyExists/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "/FailedPrecondition/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/Unimplemented/",
                     "color": "#E0B400"
                  },
                  {
                     "alias": "/InvalidArgument/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/NotFound/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/PermissionDenied/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Unauthenticated/",
                     "color": "#1F60C4"
                  },
                  {
                     "alias": "/Canceled/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DataLoss/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/DeadlineExceeded/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Internal/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OutOfRange/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/ResourceExhausted/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unavailable/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/Unknown/",
                     "color": "#C4162A"
                  },
                  {
                     "alias": "/OK/",
                     "color": "#37872D"
                  },
                  {
                     "alias": "error",
                     "color": "#C4162A"
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": {
                  "error": "#E24D42"
               },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows ratio of errors compared to the total number of handled requests from queriers.",
               "fill": 10,
               "id": 25,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 0,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows how long has it taken to handle requests from queriers, in quantiles.",
               "fill": 1,
               "id": 26,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [
                  {
                     "alias": "p99",
                     "color": "#FA6400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p90",
                     "color": "#E0B400",
                     "fill": 1,
                     "fillGradient": 1
                  },
                  {
                     "alias": "p50",
                     "color": "#37872D",
                     "fill": 10,
                     "fillGradient": 0
                  }
               ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p50 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p90 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  },
                  {
                     "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "p99 {{job}}",
                     "logBase": 10,
                     "max": null,
                     "min": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "READ - gRPC (Stream)",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "Shows the relative time of last successful upload to the object-store bucket.",
               "fill": 1,
               "id": 27,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 12,
               "stack": false,
               "steppedLine": false,
               "styles": [
                  {
                     "alias": "Time",
                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
                     "pattern": "Time",
                     "type": "hidden"
                  },
                  {
                     "alias": "Uploaded Ago",
                     "colorMode": null,
                     "colors": [ ],
                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
                     "decimals": 2,
                     "link": false,
                     "linkTargetBlank": false,
                     "linkTooltip": "Drill down",
                     "linkUrl": "",
                     "pattern": "Value",
                     "thresholds": [ ],
                     "type": "number",
                     "unit": "s"
                  },
                  {
                     "alias": "",
                     "colorMode": null,
                     "colors": [ ],
                     "dateFormat": "YYYY-MM-DD HH:mm:ss",
                     "decimals": 2,
                     "pattern": "/.*/",
                     "thresholds": [ ],
                     "type": "string",
                     "unit": "short"
                  }
               ],
               "targets": [
                  {
                     "expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=~\"$job\"})",
                     "format": "table",
                     "instant": true,
                     "intervalFactor": 2,
                     "legendFormat": "",
                     "refId": "A",
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Successful Upload",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "transform": "table",
               "type": "table",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Last Updated",
         "titleSize": "h6"
      },
      {
         "collapse": true,
         "height": "250px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 28,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "alloc all {{instance}}",
                     "legendLink": null,
                     "step": 10
                  },
                  {
                     "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "alloc heap {{instance}}",
                     "legendLink": null,
                     "step": 10
                  },
                  {
                     "expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "alloc rate all {{instance}}",
                     "legendLink": null,
                     "step": 10
                  },
                  {
                     "expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "alloc rate heap {{instance}}",
                     "legendLink": null,
                     "step": 10
                  },
                  {
                     "expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "inuse heap {{instance}}",
                     "legendLink": null,
                     "step": 10
                  },
                  {
                     "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "inuse stack {{instance}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Memory Used",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "bytes",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 29,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "go_goroutines{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Goroutines",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "id": 30,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "go_gc_duration_seconds{job=~\"$job\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{quantile}} {{instance}}",
                     "legendLink": null,
                     "step": 10
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "GC Time Quantiles",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Resources",
         "titleSize": "h6"
      }
   ],
   "schemaVersion": 14,
   "style": "dark",
   "tags": [
      "thanos-mixin"
   ],
   "templating": {
      "list": [
         {
            "current": {
               "text": "default",
               "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [ ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
         },
         {
            "allValue": null,
            "current": {
               "text": "all",
               "value": "$__all"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": "tenant",
            "multi": false,
            "name": "tenant",
            "options": [ ],
            "query": "label_values(http_requests_total{job=~\"$job\", tenant!=\"\"}, tenant)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         },
         {
            "auto": true,
            "auto_count": 300,
            "auto_min": "10s",
            "current": {
               "text": "5m",
               "value": "5m"
            },
            "hide": 0,
            "label": "interval",
            "name": "interval",
            "query": "5m,10m,30m,1h,6h,12h",
            "refresh": 2,
            "type": "interval"
         },
         {
            "allValue": null,
            "current": {
               "text": "all",
               "value": "$__all"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": "job",
            "multi": false,
            "name": "job",
            "options": [ ],
            "query": "label_values(up{job=~\".*thanos-receive.*\"}, job)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         }
      ]
   },
   "time": {
      "from": "now-1h",
      "to": "now"
   },
   "timepicker": {
      "refresh_intervals": [
         "5s",
         "10s",
         "30s",
         "1m",
         "5m",
         "15m",
         "30m",
         "1h",
         "2h",
         "1d"
      ],
      "time_options": [
         "5m",
         "15m",
         "1h",
         "6h",
         "12h",
         "24h",
         "2d",
         "7d",
         "30d"
      ]
   },
   "timezone": "UTC",
   "title": "Thanos / Receive",
   "uid": "916a852b00ccc5ed81056644718fa4fb",
   "version": 0
}