Skip to content

Commit

Permalink
PMM-12741 Percona templated alerts - interval & folder uid (#2919)
Browse files Browse the repository at this point in the history
* PMM-12741 Use folder uid for alert rules & handle evaluation interval

* PMM-12741 Format files

* PMM-12741 Rename parameter
  • Loading branch information
matejkubinec committed May 3, 2024
1 parent 3258363 commit a262bf3
Show file tree
Hide file tree
Showing 11 changed files with 173 additions and 110 deletions.
190 changes: 102 additions & 88 deletions api/managementpb/alerting/alerting.pb.go

Large diffs are not rendered by default.

29 changes: 29 additions & 0 deletions api/managementpb/alerting/alerting.pb.validate.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions api/managementpb/alerting/alerting.proto
Expand Up @@ -187,6 +187,8 @@ message CreateRuleRequest {
map<string, string> custom_labels = 8;
// Filters.
repeated Filter filters = 9;
// Evaluation Interval
google.protobuf.Duration interval = 10;
}

message CreateRuleResponse {}
Expand Down
5 changes: 5 additions & 0 deletions api/managementpb/alerting/json/alerting.json
Expand Up @@ -83,6 +83,11 @@
"type": "string",
"x-order": 2
},
"interval": {
"type": "string",
"title": "Evaluation Interval",
"x-order": 9
},
"name": {
"description": "Rule name.",
"type": "string",
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions api/swagger/swagger-dev.json
Expand Up @@ -26751,6 +26751,11 @@
}
},
"x-order": 8
},
"interval": {
"type": "string",
"title": "Evaluation Interval",
"x-order": 9
}
}
}
Expand Down
5 changes: 5 additions & 0 deletions api/swagger/swagger.json
Expand Up @@ -22017,6 +22017,11 @@
}
},
"x-order": 8
},
"interval": {
"type": "string",
"title": "Evaluation Interval",
"x-order": 9
}
}
}
Expand Down
14 changes: 6 additions & 8 deletions managed/services/grafana/client.go
Expand Up @@ -46,9 +46,8 @@ import (
var ErrFailedToGetToken = errors.New("failed to get token")

const (
defaultEvaluationInterval = time.Minute
pmmServiceTokenName = "pmm-agent-st" //nolint:gosec
pmmServiceAccountName = "pmm-agent-sa" //nolint:gosec
pmmServiceTokenName = "pmm-agent-st" //nolint:gosec
pmmServiceAccountName = "pmm-agent-sa" //nolint:gosec
)

// Client represents a client for Grafana API.
Expand Down Expand Up @@ -484,7 +483,7 @@ func (c *Client) DeleteServiceAccount(ctx context.Context, nodeName string, forc
}

// CreateAlertRule creates Grafana alert rule.
func (c *Client) CreateAlertRule(ctx context.Context, folderName, groupName string, rule *services.Rule) error {
func (c *Client) CreateAlertRule(ctx context.Context, folderUID, groupName, interval string, rule *services.Rule) error {
authHeaders, err := auth.GetHeadersFromContext(ctx)
if err != nil {
return err
Expand All @@ -497,7 +496,7 @@ func (c *Client) CreateAlertRule(ctx context.Context, folderName, groupName stri
}

var group AlertRuleGroup
if err := c.do(ctx, http.MethodGet, fmt.Sprintf("/api/ruler/grafana/api/v1/rules/%s/%s", folderName, groupName), "", authHeaders, nil, &group); err != nil {
if err := c.do(ctx, http.MethodGet, fmt.Sprintf("/api/ruler/grafana/api/v1/rules/%s/%s", folderUID, groupName), "", authHeaders, nil, &group); err != nil {
return err
}

Expand All @@ -509,8 +508,7 @@ func (c *Client) CreateAlertRule(ctx context.Context, folderName, groupName stri
group.Rules = append(group.Rules, b)

if group.Interval == "" {
// TODO: align it with grafanas default value: https://grafana.com/docs/grafana/v9.0/setup-grafana/configure-grafana/#min_interval
group.Interval = defaultEvaluationInterval.String()
group.Interval = interval
}

if err = validateDurations(group.Interval, rule.For); err != nil {
Expand All @@ -522,7 +520,7 @@ func (c *Client) CreateAlertRule(ctx context.Context, folderName, groupName stri
return err
}

if err := c.do(ctx, "POST", fmt.Sprintf("/api/ruler/grafana/api/v1/rules/%s", folderName), "", authHeaders, body, nil); err != nil {
if err := c.do(ctx, "POST", fmt.Sprintf("/api/ruler/grafana/api/v1/rules/%s", folderUID), "", authHeaders, body, nil); err != nil {
if err != nil {
if cErr, ok := errors.Cause(err).(*clientError); ok { //nolint:errorlint
return status.Error(codes.InvalidArgument, cErr.ErrorMessage)
Expand Down
2 changes: 1 addition & 1 deletion managed/services/management/alerting/deps.go
Expand Up @@ -24,7 +24,7 @@ import (
)

type grafanaClient interface {
CreateAlertRule(ctx context.Context, folderName, groupName string, rule *services.Rule) error
CreateAlertRule(ctx context.Context, folderUID, groupName, interval string, rule *services.Rule) error
GetDatasourceUIDByID(ctx context.Context, id int64) (string, error)
GetFolderByUID(ctx context.Context, uid string) (*gapi.Folder, error)
}
10 changes: 5 additions & 5 deletions managed/services/management/alerting/mock_grafana_client_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 10 additions & 8 deletions managed/services/management/alerting/service.go
Expand Up @@ -52,8 +52,9 @@ import (
)

const (
templatesDir = "/srv/alerting/templates"
portalRequestTimeout = 2 * time.Minute // time limit to get templates list from the portal
templatesDir = "/srv/alerting/templates"
portalRequestTimeout = 2 * time.Minute // time limit to get templates list from the portal
defaultEvaluationInterval = time.Minute

dirPerm = os.FileMode(0o775)
)
Expand Down Expand Up @@ -701,11 +702,6 @@ func (s *Service) CreateRule(ctx context.Context, req *alerting.CreateRuleReques
return nil, status.Error(codes.InvalidArgument, "Rule group name should be specified.")
}

folder, err := s.grafanaClient.GetFolderByUID(ctx, req.FolderUid)
if err != nil {
return nil, err
}

metricsDatasourceUID, err := s.grafanaClient.GetDatasourceUIDByID(ctx, 1) // 1 - it's id of Metrics datasource in PMM
if err != nil {
return nil, err
Expand Down Expand Up @@ -803,7 +799,13 @@ func (s *Service) CreateRule(ctx context.Context, req *alerting.CreateRuleReques
Labels: labels,
}

err = s.grafanaClient.CreateAlertRule(ctx, folder.Title, req.Group, &rule)
// TODO: align it with grafanas default value: https://grafana.com/docs/grafana/v9.0/setup-grafana/configure-grafana/#min_interval
interval := defaultEvaluationInterval.String()
if req.Interval != nil {
interval = req.Interval.AsDuration().String()
}

err = s.grafanaClient.CreateAlertRule(ctx, req.FolderUid, req.Group, interval, &rule)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit a262bf3

Please sign in to comment.