Skip to content

feature: Grafana alert for catching Neutron IPAM Errors while attempt… #1054

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 24, 2025
Merged

feature: Grafana alert for catching Neutron IPAM Errors while attempt… #1054

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dd16127
feature: Grafana alert for catching Neutron IPAM Errors while attempt…
anande Jun 23, 2025
3b0695b
feature: Grafana alert for catching Neutron IPAM Errors while attempt…
anande Jun 24, 2025
5bd1166
feature: Grafana alert for catching Neutron IPAM Errors while attempt…
anande Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 126 additions & 76 deletions base-helm-configs/grafana/grafana-helm-overrides.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,93 +38,143 @@ grafana.ini:
type: mysql
host: mariadb-cluster.grafana.svc:3306
user: $__file{/etc/secrets/grafana-db/username}
password: $__file{/etc/secrets/grafana-db/password}
password: $__file{/etc/secrets/grafana-db/password}
name: grafana

datasources:
datasources.yaml:
apiversion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090
isdefault: true
- name: Loki
type: loki
access: proxy
url: http://loki-gateway.{{ $.Release.Namespace }}.svc.cluster.local:80
editable: false
- name: Prometheus
type: prometheus
access: proxy
url: http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090
isdefault: true
- name: Loki
type: loki
access: proxy
url: http://loki-gateway.{{ $.Release.Namespace }}.svc.cluster.local:80
editable: false

alerting:
rules.yaml:
groups:
- orgId: 1
name: loki 1 min eval
folder: rules
interval: 1m
rules:
- uid: ba943125-33ca-4e4e-85f8-13359a8e4d65
title: OVN claim storm
condition: B
data:
- refId: A
- orgId: 1
name: loki 1 min eval
folder: rules
interval: 1m
rules:
- uid: ba943125-33ca-4e4e-85f8-13359a8e4d65
title: OVN claim storm
condition: B
data:
- refId: A
queryType: instant
relativeTimeRange:
from: 60
to: 0
datasourceUid: P8E80F9AEF21F6940
model:
editorMode: builder
expr: rate({app="ovs"} |= `binding|INFO|cr-lrp` [1m])
intervalMs: 60000
maxDataPoints: 43200
queryType: instant
refId: A
- refId: B
relativeTimeRange:
from: 60
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
refId: B
type: threshold
noDataState: OK
execErrState: Error
for: 0s
notifications:
- uid: prom-alertmanager-notification
annotations:
description: >-
Checks app=ovs (ovs-ovn) pod logs for lines with string
'binding|INFO|cr-lrp'
summary: >-
This alerts on rapid port claims for cr-lrp ports on OVN
gateway nodes, which overloads the OVN south database and
interferes with the function of the affected ports.
labels: {}
isPaused: false
# Generated UUID using 'uuidgen'
- uid: c14dd8fd-54ec-4e15-9813-e02cc3269899
title: Neutron IPAM Duplicate Entry Error
condition: C
data:
- refId: A
queryType: instant
relativeTimeRange:
from: 60
to: 0
# Using same loki datasource as rule#ba943125-33ca-4e4e-85f8-13359a8e4d65
datasourceUid: P8E80F9AEF21F6940
model:
expr: rate({app="fluentbit"} |= `Duplicate entry|ERROR` [1m])
queryType: instant
relativeTimeRange:
from: 60
to: 0
datasourceUid: P8E80F9AEF21F6940
model:
editorMode: builder
expr: rate({app="ovs"} |= `binding|INFO|cr-lrp` [1m])
intervalMs: 60000
maxDataPoints: 43200
queryType: instant
refId: A
- refId: B
relativeTimeRange:
from: 60
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
refId: B
type: threshold
noDataState: OK
execErrState: Error
for: 0s
notifications:
- uid: prom-alertmanager-notification
annotations:
description: >-
Checks app=ovs (ovs-ovn) pod logs for lines with string
'binding|INFO|cr-lrp'
summary: >-
This alerts on rapid port claims for cr-lrp ports on OVN
gateway nodes, which overloads the OVN south database and
interferes with the function of the affected ports.
labels: {}
isPaused: false
refId: A
- refId: B
relativeTimeRange:
# Past 60 seconds (can be adjusted further)
from: 60
# 0 denotes till current time
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1
- 0
type: gt
operator:
type: and
reducer:
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: A
refId: B
type: threshold
noDataState: OK
execErrState: Error
notifications:
- uid: prom-alertmanager-notification
annotations:
summary: >
Checks for log lines containing 'Duplicate entry|ERROR' in nova logs.
isPaused: false
contactpoints.yaml:
secret:
apiVersion: 1
Expand Down