prom-rules-tests.yaml

---
rule_files:
  - /tmp/rules.verify

group_eval_order:
  - recordingRules.rules
  - alerts.rules
#information about this format can be found in: https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/
tests:
  # Alerts to test whether our operators are up or not
  - interval: 1m
    input_series:
      - series: 'up{namespace="ci", pod="virt-api-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"
      - series: 'up{namespace="ci", pod="virt-controller-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"
      - series: 'up{namespace="ci", pod="virt-operator-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"

    alert_rule_test:
      # it must not trigger before 10m
      - eval_time: 8m
        alertname: VirtAPIDown
        exp_alerts: []
      - eval_time: 8m
        alertname: VirtControllerDown
        exp_alerts: [ ]
      - eval_time: 8m
        alertname: VirtOperatorDown
        exp_alerts: [ ]
      # it must trigger when there is no data
      - eval_time: 10m
        alertname: VirtAPIDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-api servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtAPIDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 10m
        alertname: VirtControllerDown
        exp_alerts:
          - exp_annotations:
              summary: "No running virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 10m
        alertname: VirtOperatorDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-operator servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      # it must trigger when operators are not healthy
      - eval_time: 16m
        alertname: VirtAPIDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-api servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtAPIDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 16m
        alertname: VirtControllerDown
        exp_alerts:
          - exp_annotations:
              summary: "No running virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 16m
        alertname: VirtOperatorDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-operator servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorDown"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      # it must not trigger when operators are healthy
      - eval_time: 17m
        alertname: VirtAPIDown
        exp_alerts: []
      - eval_time: 17m
        alertname: VirtControllerDown
        exp_alerts: [ ]
      - eval_time: 17m
        alertname: VirtOperatorDown
        exp_alerts: [ ]

    # Alert to test when there are VMIs running on a node with an unready virt-handler pod
    # Alert should not fire for node with no running VMIs.
  - interval: 1m
    input_series:
      - series: 'kube_pod_info{pod="virt-handler-asdf", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdf", condition="true"}'
        values: '0 0 0 0 0 0 0 0 0 0 0'
      - series: 'kube_pod_info{pod="virt-launcher-testvm-123", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-asdfg", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdfg", condition="true"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-launcher-vmi", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-abcd", node="node03"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-abcd", condition="true"}'
        values: '0 0 0 0 0 0 0 0 0 0 0'
      - series: 'kube_pod_info{pod="virt-launcher-novmi", node="node03"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-handler pod detected on node node01 with running vmis for more than 10 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/OrphanedVirtualMachineInstances"
            exp_labels:
              node: "node01"
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

    # Alert to test when there are VMIs running on a node without a virt-handler pod
    # Alert should not fire for node with no running VMIs.
  - interval: 1m
    input_series:
      - series: 'kube_pod_info{pod="virt-handler-asdf", node="node01"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdf", condition="true"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_info{pod="virt-launcher-testvm-123", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-asdfg", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdfg", condition="true"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-launcher-vmi", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-abcd", node="node03"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_status_ready{pod="virt-handler-abcd", condition="true"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_info{pod="virt-launcher-novmi", node="node03"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'


    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-handler pod detected on node node01 with running vmis for more than 10 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/OrphanedVirtualMachineInstances"
            exp_labels:
              node: "node01"
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Some virt controllers are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready_status{namespace="ci", pod="virt-controller-1"}'
        values: '1+0x11'
      - series: 'kubevirt_virt_controller_ready_status{namespace="ci", pod="virt-controller-2"}'
        values: '0+0x11'
      - series: 'up{namespace="ci", pod="virt-controller-1"}'
        values: '1+0x11'
      - series: 'up{namespace="ci", pod="virt-controller-2"}'
        values: '1+0x11'

    alert_rule_test:
      - eval_time: 10m
        alertname: LowReadyVirtControllersCount
        exp_alerts:
          - exp_annotations:
              summary: "Some virt controllers are running but not ready."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowReadyVirtControllersCount"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt controllers are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready_status{namespace="ci", pod="virt-controller-1"}'
        values: "0 0 0 0 0 0 0 0 0 0 0"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtController
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtController
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtController"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
  # All virt controllers are not ready (ImagePullBackOff)
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready_status{namespace="ci", pod="virt-controller-1"}'
        values: "stale stale stale stale stale stale stale stale stale stale"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtController
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtController
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtController"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt operators are not ready (ImagePullBackOff)
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_operator_ready_status{namespace="ci", pod="virt-operator-1"}'
        values: "stale stale stale stale stale stale stale stale stale stale"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtOperator
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtOperator
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-operator was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtOperator"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt operators are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_operator_ready_status{namespace="ci", pod="virt-operator-1"}'
        values: "0 0 0 0 0 0 0 0 0 0 0"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtOperator
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtOperator
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-operator was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtOperator"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Burst REST errors
  # values: '0+10x20' == values: "0 10 20 30 40 ... 190"
  # values: '0+100x20' == values :"0 100 200 .... 1900"
  # so now for EACH POD the total requests should be 10+20+30+..+190+100+200+300+...+1900
  # and the number of requests with error code should be 190+100+200+300+...+1900 which is more than 80% of the total requests
  # in each 5 minutes interval of the test and the error condition(more than 80% of the requests has error code) is true for
  # more than 5 minutes(because each test run for 20 minutes) which should fire an alert for EACH POD
  # values : 0+100x15  0+100x5  ==  :"0 100 200 .... 1400  0 100 200 300 400"  we should treat values : `0+100x20` and
  # values : `0+100x15  0+100x5`  the same way because prometheus counters might reset
  - interval: 1m
    input_series:
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="200"}'
        values: '0+10x20'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="400"}'
        values: '0+100x15  0+100x5'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="200"}'
        values: '0+10x20'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="400"}'
        values: '0+100x15  0+100x5'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="200"}'
        values: '0+10x20'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="500"}'
        values: '0+100x15  0+100x5'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-api-1", code="200"}'
        values: '0+10x20'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-api-1", code="500"}'
        values: '0+100x15  0+100x5'

    alert_rule_test:
      - eval_time: 20m
        alertname: VirtControllerRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-controller for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtOperatorRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-operator for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtHandlerRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-handler for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtHandlerRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtApiRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-api for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtApiRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              operator_health_impact: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # values: '0+10x100' == values: "0 10 20 30 40 ... 990"
  # values: '0+5x100' == values :"0 5 10 .... 495"
  # so now for EACH POD the total requests should be 10+20+30+..+990+5+10+15+...+495
  # and the number of requests with error code should be 5+10+15+...+495 which is more than 5% of the total requests
  # the error condition(more than 5% of the requests has error code) is true which should fire an alert for EACH POD
  # High REST errors
  # values : '0+5x90 0+5x10'  ==  :"0 5 10 .... 445  0 5 10 ... 45"  we should treat values : '0+5x100' and
  # values : '0+5x90 0+5x10'  the same way because prometheus counters might reset
  - interval: 1m
    input_series:
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="200"}'
        values: '0+10x100'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="400"}'
        values: '0+5x90 0+5x10'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="200"}'
        values: '0+10x100'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="400"}'
        values: '0+5x90 0+5x10'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="200"}'
        values: '0+10x100'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="500"}'
        values: '0+5x90 0+5x10'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-api-1", code="200"}'
        values: '0+10x100'
      - series: 'kubevirt_rest_client_requests_total{namespace="ci", pod="virt-api-1", code="500"}'
        values: '0+5x90 0+5x10'

    alert_rule_test:
      - eval_time: 100m
        alertname: VirtControllerRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-controller for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtOperatorRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-operator for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtHandlerRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-handler for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtHandlerRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtApiRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-api for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtApiRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Some nodes without KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "110 110 110 110 110 110"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "0 0 0 0 0 0"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts:
          - exp_annotations:
              description: "Low number of nodes with KVM resource available."
              summary: "At least two nodes with kvm resource required for VM live migration."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowKVMNodesCount"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All nodes without KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "0 0 0 0 0 0"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "0 0 0 0 0 0"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts:
          - exp_annotations:
              description: "Low number of nodes with KVM resource available."
              summary: "At least two nodes with kvm resource required for VM live migration."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowKVMNodesCount"
            exp_labels:
              severity: "warning"
              operator_health_impact: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Two nodes with KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "110 110 110 110 110 110"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "110 110 110 110 110 110"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts: []

  # Memory utilization less than 50MB close to requested memory - based on memory working set
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "17185920 18234496 18234496 19283072"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 50Mi and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization less than 50MB close to requested memory - based on memory RSS
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "17185920 18234496 18234496 19283072"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 50Mi and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization less than 50MB close to requested memory - based on memory RSS and memory working set
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "17185920 18234496 18234496 19283072"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "17185920 18234496 18234496 19283072"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 50Mi and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization more than 50MB close to requested memory
  - interval: 30s
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "9922944 8874368 8874368 7825792"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "9922944 8874368 8874368 7825792"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts: []

  # Test recording rule
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-example-1", namespace="namespace-example-1",container="compute", resource="memory"}'
        # time:  0          1          2          3
        values: "1376882688 1376882688 1376882688 1376882688"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-example-1", namespace="namespace-example-1",container="compute", resource="memory", prometheus_replica="prometheus-k8s-0"}'
        # time:  0          1          2          3
        values: "1073176576 1073176576 1073176576 1273176576"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-example-1", namespace="namespace-example-1",container="compute", resource="memory", prometheus_replica="prometheus-k8s-1"}'
        # time:  0          1          2          3
        values: "1073176576 1073176576 1073176576 1273176576"
    promql_expr_test:
      - expr: 'kubevirt_vm_container_free_memory_bytes_based_on_working_set_bytes'
        eval_time: 1m
        exp_samples:
          - labels: 'kubevirt_vm_container_free_memory_bytes_based_on_working_set_bytes{pod="virt-launcher-example-1", namespace="namespace-example-1",container="compute"}'
            value: 303706112
      - expr: 'kubevirt_vm_container_free_memory_bytes_based_on_working_set_bytes'
        eval_time: 3m
        exp_samples:
          - labels: 'kubevirt_vm_container_free_memory_bytes_based_on_working_set_bytes{pod="virt-launcher-example-1", namespace="namespace-example-1",container="compute"}'
            value: 103706112

  # VM eviction strategy is set but vm is not migratable
  - interval: 1m
    input_series:
      - series: 'kubevirt_vmi_non_evictable{node="node1", namespace="ns-test", name="vm-evict-nonmigratable"}'
        values: "1 1 1 1 1 1 1 1"

    alert_rule_test:
      - eval_time: 1m
        alertname: VMCannotBeEvicted
        exp_alerts:
          - exp_annotations:
              description: "Eviction policy for VirtualMachine vm-evict-nonmigratable in namespace ns-test (on node node1) is set to Live Migration but the VM is not migratable"
              summary: "The VM's eviction strategy is set to Live Migration but the VM is not migratable"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VMCannotBeEvicted"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              name: "vm-evict-nonmigratable"
              namespace: "ns-test"
              node: "node1"

  # VM eviction strategy is set and vm is migratable
  - interval: 1m
    input_series:
      - series: 'kubevirt_vmi_non_evictable{node="node1", namespace="ns-test", name="vm-evict-migratable"}'
        values: "0 0 0 0 0 0 0 0 "

    alert_rule_test:
      - eval_time: 1m
        alertname: VMCannotBeEvicted
        exp_alerts: []

  # Test recording rule
  - interval: 1m
    input_series:
      # In reality there are many labels on these metrics
      # they are the same except the ones containing vm name like "name" in the example below
      - series: 'kubevirt_vmi_memory_available_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "1376882688 1376882688 1376882688 1376882688"
      - series: 'kubevirt_vmi_memory_available_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "2893266944 2893266944 2893266944 2893266944"
      - series: 'kubevirt_vmi_memory_usable_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "1073176576 1073176576 1073176576 1273176576"
      - series: 'kubevirt_vmi_memory_usable_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "2448936960 2448936960 2448936960 2658936964"
    promql_expr_test:
      - expr: 'kubevirt_vmi_memory_used_bytes'
        eval_time: 1m
        exp_samples:
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
            value: 303706112
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
            value: 444329984
      - expr: 'kubevirt_vmi_memory_used_bytes'
        eval_time: 3m
        exp_samples:
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
            value: 103706112
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
            value: 234329980

  # Excessive VMI Migrations in a period of time
  - interval: 1h
    input_series:
      - series: 'kubevirt_vmi_migration_succeeded{vmi="vmi-example-1", namespace="namespace-example-1"}'
        # time:  0 1 2 3 4 5
        values: "_ _ _ 1 7 13"

    alert_rule_test:
      # at 4h, there are total of 11 migrations made on a single VMI, so the alert should not be fired.
      - eval_time: 4h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts: []
      # at 5h, there are total of 13 migrations made on a single VMI, thus the alert is expected to be fired.
      - eval_time: 5h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts:
          - exp_annotations:
              description: "VirtualMachineInstance vmi-example-1 in namespace namespace-example-1 has been migrated more than 12 times during the last 24 hours"
              summary: "An excessive amount of migrations have been detected on a VirtualMachineInstance in the last 24 hours."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtVMIExcessiveMigrations"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              vmi: vmi-example-1
              namespace: namespace-example-1
      - eval_time: 24h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts:
          - exp_annotations:
              description: "VirtualMachineInstance vmi-example-1 in namespace namespace-example-1 has been migrated more than 12 times during the last 24 hours"
              summary: "An excessive amount of migrations have been detected on a VirtualMachineInstance in the last 24 hours."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtVMIExcessiveMigrations"
            exp_labels:
              severity: "warning"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              vmi: vmi-example-1
              namespace: namespace-example-1
      # will need to evaluate 24h after the alert is triggered to disregard the increases and clear the alert.
      - eval_time: 30h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts: []

  # No nodes are available to host VMs
  - interval: 1m
    input_series:
      - series: 'kube_node_labels{node="node01",label_kubevirt_io_schedulable="true"}'
        values: "_ _ _ _ _ _ _ 1+0x15"
      - series: 'kube_node_status_allocatable{node="node01", resource="devices_kubevirt_io_kvm"}'
        values: "1000+0x15 0+0x7"
      - series: 'kubevirt_configuration_emulation_enabled'
        values: "0+0x22"

    alert_rule_test:
      # has kvm allocatable but no schedulable label
      - eval_time: 6m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts:
          - exp_annotations:
              summary: "There are no available nodes in the cluster to run VMs."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtNoAvailableNodesToRunVMs"
            exp_labels:
              severity: "warning"
              kubernetes_operator_component: "kubevirt"
              kubernetes_operator_part_of: "kubevirt"
              operator_health_impact: "critical"

      # has kvm allocatable and schedulable label
      - eval_time: 7m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts: []

      # has schedulable label but no kvm allocatable nor emulation enabled
      - eval_time: 21m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts:
          - exp_annotations:
              summary: "There are no available nodes in the cluster to run VMs."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtNoAvailableNodesToRunVMs"
            exp_labels:
              severity: "warning"
              kubernetes_operator_component: "kubevirt"
              kubernetes_operator_part_of: "kubevirt"
              operator_health_impact: "critical"

  - interval: 1m
    input_series:
      - series: 'kube_node_labels{node="node01",label_kubevirt_io_schedulable="true"}'
        values: "0+0x7 1+0x14"
      - series: 'kube_node_status_allocatable{node="node01", resource="devices_kubevirt_io_kvm"}'
        values: "0+0x21"
      - series: 'kubevirt_configuration_emulation_enabled'
        values: "0+0x7 1+0x7 0+0x7"

    alert_rule_test:
      # no schedulable label
      - eval_time: 6m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts:
          - exp_annotations:
              summary: "There are no available nodes in the cluster to run VMs."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtNoAvailableNodesToRunVMs"
            exp_labels:
              severity: "warning"
              kubernetes_operator_component: "kubevirt"
              kubernetes_operator_part_of: "kubevirt"
              operator_health_impact: "critical"

      # has schedulable label and emulation enabled
      - eval_time: 8m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts: []

      # has schedulable label, but no kvm allocatable and no emulation enabled
      - eval_time: 21m
        alertname: KubeVirtNoAvailableNodesToRunVMs
        exp_alerts:
          - exp_annotations:
              summary: "There are no available nodes in the cluster to run VMs."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtNoAvailableNodesToRunVMs"
            exp_labels:
              severity: "warning"
              kubernetes_operator_component: "kubevirt"
              kubernetes_operator_part_of: "kubevirt"
              operator_health_impact: "critical"

  # Deprecated APIs being requested.
  - interval: 1m
    input_series:
      - series: 'apiserver_requested_deprecated_apis{resource="virtualmachines", group="kubevirt.io", version="v1alpha3"}'
        values: '0 0 1'
      - series: 'apiserver_request_total{resource="virtualmachines", group="kubevirt.io", version="v1alpha3"}'
        values: '0 0 1 2'

    alert_rule_test:
      - eval_time: 1m
        alertname: KubeVirtDeprecatedAPIRequested
        exp_alerts: []
      - eval_time: 2m
        alertname: KubeVirtDeprecatedAPIRequested
        exp_alerts:
          - exp_annotations:
              description: "Detected requests to the deprecated virtualmachines.kubevirt.io/v1alpha3 API."
              summary: "Detected 1 requests in the last 10 minutes."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtDeprecatedAPIRequested"
            exp_labels:
              severity: "info"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              resource: "virtualmachines"
              group: "kubevirt.io"
              version: "v1alpha3"
      - eval_time: 3m
        alertname: KubeVirtDeprecatedAPIRequested
        exp_alerts:
          - exp_annotations:
              description: "Detected requests to the deprecated virtualmachines.kubevirt.io/v1alpha3 API."
              summary: "Detected 2 requests in the last 10 minutes."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtDeprecatedAPIRequested"
            exp_labels:
              severity: "info"
              operator_health_impact: "none"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              resource: "virtualmachines"
              group: "kubevirt.io"
              version: "v1alpha3"
      - eval_time: 13m
        alertname: KubeVirtDeprecatedAPIRequested
        exp_alerts: []