prometheus-alerts.yaml

---
groups:
- name: logging_loki.alerts
  rules:
  - alert: LokiRequestErrors
    annotations:
      message: |-
        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
      summary: "At least 10% of requests are responded by 5xx server errors."
      runbook_url: "[[ .RunbookURL ]]#Loki-Request-Errors"
    expr: |
      sum(
        job_namespace_route_statuscode:loki_request_duration_seconds_count:irate1m{status_code=~"5.."}
      ) by (job, namespace, route)
      /
      sum(
        job_namespace_route_statuscode:loki_request_duration_seconds_count:irate1m
      ) by (job, namespace, route)
      * 100
      > 10
    for: 15m
    labels:
      severity: critical
  - alert: LokiStackWriteRequestErrors
    annotations:
      message: |-
        {{ printf "%.2f" $value }}% of write requests from {{ $labels.job }} in {{ $labels.namespace }} are returned with server errors.
      summary: "At least 10% of write requests to the lokistack-gateway are responded with 5xx server errors."
      runbook_url: "[[ .RunbookURL ]]#LokiStack-Write-Request-Errors"
    expr: |
      sum(
        code_handler_job_namespace:lokistack_gateway_http_requests:irate1m{code=~"5..", handler="push"}
      ) by (job, namespace)
      /
      sum(
        code_handler_job_namespace:lokistack_gateway_http_requests:irate1m{handler="push"}
      ) by (job, namespace)
      * 100
      > 10
    for: 15m
    labels:
      severity: critical
  - alert: LokiStackReadRequestErrors
    annotations:
      message: |-
        {{ printf "%.2f" $value }}% of query requests from {{ $labels.job }} in {{ $labels.namespace }} are returned with server errors.
      summary: "At least 10% of query requests to the lokistack-gateway are responded with 5xx server errors."
      runbook_url: "[[ .RunbookURL ]]#LokiStack-Read-Request-Errors"
    expr: |
      sum(
        code_handler_job_namespace:lokistack_gateway_http_requests:irate1m{code=~"5..", handler=~"query|query_range|label|labels|label_values"}
      ) by (job, namespace)
      /
      sum(
        code_handler_job_namespace:lokistack_gateway_http_requests:irate1m{handler=~"query|query_range|label|labels|label_values"}
      ) by (job, namespace)
      * 100
      > 10
    for: 15m
    labels:
      severity: critical
  - alert: LokiRequestPanics
    annotations:
      message: |-
        {{ $labels.job }} is experiencing an increase of {{ $value }} panics.
      summary: "A panic was triggered."
      runbook_url: "[[ .RunbookURL ]]#Loki-Request-Panics"
    expr: |
      sum(
        increase(
          loki_panic_total[10m]
        )
      ) by (job, namespace)
      > 0
    labels:
      severity: critical
  - alert: LokiRequestLatency
    annotations:
      message: |-
        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
      summary: "The 99th percentile is experiencing high latency (higher than 1 second)."
      runbook_url: "[[ .RunbookURL ]]#Loki-Request-Latency"
    expr: |
      histogram_quantile(0.99,
        sum(
          irate(
            loki_request_duration_seconds_bucket{route!~"(?i).*tail.*"}[1m]
          )
        ) by (job, le, namespace, route)
      )
      > 1
    for: 15m
    labels:
      severity: critical
  - alert: LokiTenantRateLimit
    annotations:
      message: |-
        {{ $labels.job }} {{ $labels.route }} is experiencing 429 errors.
      summary: "At least 10% of requests are responded with the rate limit error code."
      runbook_url: "[[ .RunbookURL ]]#Loki-Tenant-Rate-Limit"
    expr: |
      sum(
        job_namespace_route_statuscode:loki_request_duration_seconds_count:irate1m{status_code="429"}
      ) by (job, namespace, route)
      /
      sum(
        job_namespace_route_statuscode:loki_request_duration_seconds_count:irate1m
      ) by (job, namespace, route)
      * 100
      > 10
    for: 15m
    labels:
      severity: warning
  - alert: LokiStorageSlowWrite
    annotations:
      message: |-
        The storage path is experiencing slow write response rates.
      summary: "The storage path is experiencing slow write response rates."
      runbook_url: "[[ .RunbookURL ]]#Loki-Storage-Slow-Write"
    expr: |
      histogram_quantile(0.99,
        sum(
          job_le_namespace_operation:loki_boltdb_shipper_request_duration_seconds_bucket:rate5m{operation="WRITE"}
        ) by (job, le, namespace)
      )
      > 1
    for: 15m
    labels:
      severity: warning
  - alert: LokiStorageSlowRead
    annotations:
      message: |-
        The storage path is experiencing slow read response rates.
      summary: "The storage path is experiencing slow read response rates."
      runbook_url: "[[ .RunbookURL ]]#Loki-Storage-Slow-Read"
    expr: |
      histogram_quantile(0.99,
        sum(
          job_le_namespace_operation:loki_boltdb_shipper_request_duration_seconds_bucket:rate5m{operation="Shipper.Query"}
        ) by (job, le, namespace)
      )
      > 5
    for: 15m
    labels:
      severity: warning
  - alert: LokiWritePathHighLoad
    annotations:
      message: |-
        The write path is experiencing high load.
      summary: "The write path is experiencing high load, causing backpressure storage flushing."
      runbook_url: "[[ .RunbookURL ]]#Loki-Write-Path-High-Load"
    expr: |
      sum(
        loki_ingester_wal_replay_flushing
      ) by (job, namespace)
      > 0
    for: 15m
    labels:
      severity: warning
  - alert: LokiReadPathHighLoad
    annotations:
      message: |-
        The read path is experiencing high load.
      summary: "The read path has high volume of queries, causing longer response times."
      runbook_url: "[[ .RunbookURL ]]#Loki-Read-Path-High-Load"
    expr: |
      histogram_quantile(0.99,
        sum(
          rate(
            loki_logql_querystats_latency_seconds_bucket[5m]
          )
        ) by (job, le, namespace)
      )
      > 30
    for: 15m
    labels:
      severity: warning
  - alert: LokistackSchemaUpgradesRequired
    annotations:
      message: |-
        The LokiStack "{{ $labels.stack_name }}" in namespace "{{ $labels.stack_namespace }}" is using a storage schema
        configuration that does not contain the latest schema version. It is recommended to update the schema
        configuration to update the schema version to the latest version in the future.
      summary: "One or more of the deployed LokiStacks contains an outdated storage schema configuration."
      runbook_url: "[[ .RunbookURL ]]#Lokistack-Schema-Upgrades-Required"
    expr: |
      sum (
        lokistack_status_condition{reason="StorageNeedsSchemaUpdate",status="true"}
      ) by (stack_namespace, stack_name)
      > 0
    for: 1m
    labels:
      severity: warning