VictoriaTraces/deployment/docker/rules/alerts-vtraces.yml at master · VictoriaMetrics/VictoriaTraces · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# File contains VictoriaTraces-specific alerts for VictoriaTraces installations.
# The alerts below are just recommendations and may require some updates
# and threshold calibration according to every specific setup.
groups:
  - name: vtraces
    rules:
      - alert: DiskRunsOutOfSpace
        expr: |
          sum(vt_data_size_bytes) by(job, instance) /
          (
           sum(vt_free_disk_space_bytes) by(job, instance) +
           sum(vt_data_size_bytes) by(job, instance)
          ) > 0.8
        for: 30m
        labels:
          severity: critical
        annotations:
          summary: |-
            Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon
          description: |-
            Disk utilisation on instance {{ $labels.instance }} is more than 80%.
             Having less than 20% of free disk space could cripple merge processes and overall performance. Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
      - alert: ConcurrentQueriesHitTheLimit
        expr: avg_over_time(vt_concurrent_select_current[1m]) >= vt_concurrent_select_capacity
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: |-
            {{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent query limit
          description: |-
            The limit of concurrent queries on instance {{ $labels.instance }} is controlled by `-search.maxConcurrentRequests`.
            Usually, when the component constantly hits the limit it is likely overloaded and requires more CPU or more replicas.
            In some cases the alert might trigger if clients send too many expensive queries to the component.
            If the component's CPU usage and storage latency are at normal level, then
            it might be worth adjusting `-search.maxConcurrentRequests`.
      - alert: TooHighQueryLoad
        expr: increase(vt_concurrent_select_limit_timeout_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: |-
            Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }}
          description: |-
            Instance {{ $labels.instance }} ({{ $labels.job }}) is failing to serve read queries during last 15m.
            Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were
            put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren't served.
            This happens if instance is overloaded with the current workload, or datasource is too slow to respond.
            Possible solutions are the following:
            * reduce the query load;
            * increase compute resources or number of replicas;
            * adjust limits `-search.maxConcurrentRequests` and `-search.maxQueueDuration`.
            See more at https://docs.victoriametrics.com/victoriatraces/troubleshooting/.