Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Top‑level packages worth knowing:
- `cmd/alertmanager/` — main binary entry point (`main.go`); thin wrapper that parses flags and calls `app`.
- `app/` — embeddable Alertmanager runtime extracted from `cmd/alertmanager`. Owns the process lifecycle (`New`/`Start`/`Stop`/`Reload`/`Run`), subsystem wiring (`setup`), config-reload subgraph (`reloader`), listeners and `Options`. Lets tests and other binaries run Alertmanager in‑process. See https://github.com/prometheus/alertmanager/issues/406.
- `cmd/amtool/` — CLI for interacting with the Alertmanager API.
- `api/` — HTTP API. `api/v2/` is the active API; `api/v1_deprecation_router.go` only returns deprecation responses.
- `api/` — HTTP API. `api/v2/` is the active REST API; `api/connect/` is the experimental ConnectRPC API mounted under `/api/`.
- `cli/` — `amtool` command implementations.
- `cluster/` — HA gossip clustering (memberlist-based).
- `config/` — YAML config parsing, validation, secrets, coordinator.
Expand Down Expand Up @@ -141,7 +141,7 @@ If you regenerate API code or UI data, commit the regenerated files alongside th

## Things to avoid

- Don't reintroduce APIv1; it was removed in 0.27.0. The only thing left under `api/v1_deprecation_router.go` is a deprecation responder.
- Don't reintroduce APIv1; it was removed in 0.27.0 and its deprecation responder has since been dropped.
- Don't bypass the notification pipeline (`notify/notify.go`) — new stages should be added as `notify.Stage` implementations.
- Don't add new top‑level dependencies without checking `.golangci.yml` `depguard` rules and `go.mod` minimums.
- Don't commit generated artifacts unless the corresponding source (`openapi.yaml`, `email.html`, etc.) changed in the same commit.
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
## main / (unreleased)

* [CHANGE] api: Remove the `/api/v1/` deprecation responder. These endpoints have returned `410 Gone` since 0.27.0; requests to `/api/v1/` are no longer handled.
* [FEATURE] api: Add an experimental ConnectRPC API served alongside `/api/v2/` under the version-neutral `/api/` prefix, exposing the Connect, gRPC, and gRPC-Web protocols plus the gRPC Health Checking Protocol and server reflection. The first service is `status.v3.StatusService`.
* [CHANGE] notify: The `reason` label on `alertmanager_notifications_failed_total` now distinguishes `authError` (HTTP 401/403) and `rateLimited` (HTTP 429) from the generic `clientError`. Dashboards/alerts matching `reason="clientError"` for these codes must be updated.
* [ENHANCEMENT] notify: The discord and webex integrations now report a failure `reason` on `alertmanager_notifications_failed_total`.
* [BUGFIX] webhook: Keep custom `payload` string values verbatim instead of reinterpreting JSON leaves that look like YAML (e.g. values ending with a colon). #5302
Expand Down
33 changes: 26 additions & 7 deletions api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/prometheus/common/route"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"

apiconnect "github.com/prometheus/alertmanager/api/connect"
apiv2 "github.com/prometheus/alertmanager/api/v2"
"github.com/prometheus/alertmanager/cluster"
"github.com/prometheus/alertmanager/config"
Expand All @@ -41,8 +42,8 @@ import (

// API represents all APIs of Alertmanager.
type API struct {
v2 *apiv2.API
deprecationRouter *V1DeprecationRouter
v2 *apiv2.API
connect *apiconnect.API

requestDuration *prometheus.HistogramVec
requestsInFlight prometheus.Gauge
Expand Down Expand Up @@ -116,6 +117,7 @@ func New(opts Options) (*API, error) {
concurrency = max(runtime.GOMAXPROCS(0), 8)
}

// The Connect API is always mounted alongside API v2.
v2, err := apiv2.NewAPI(
opts.Alerts,
opts.GroupFunc,
Expand All @@ -128,6 +130,7 @@ func New(opts Options) (*API, error) {
if err != nil {
return nil, err
}
connect := apiconnect.NewAPI(opts.Peer, l.With("api", "connect"))

requestsInFlight := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "alertmanager_http_requests_in_flight",
Expand All @@ -149,8 +152,8 @@ func New(opts Options) (*API, error) {
}

return &API{
deprecationRouter: NewV1DeprecationRouter(l.With("version", "v1")),
v2: v2,
connect: connect,
requestDuration: opts.RequestDuration,
requestsInFlight: requestsInFlight,
concurrencyLimitExceeded: concurrencyLimitExceeded,
Expand All @@ -167,16 +170,14 @@ func New(opts Options) (*API, error) {
// true for the concurrency limit, with the exception that it is only applied to
// GET requests.
func (api *API) Register(r *route.Router, routePrefix string) *http.ServeMux {
// TODO(gotjosh) API V1 was removed as of version 0.27, when we reach 1.0.0 we should removed these deprecation warnings.
api.deprecationRouter.Register(r.WithPrefix("/api/v1"))

mux := http.NewServeMux()
mux.Handle("/", api.limitHandler(r))

apiPrefix := ""
if routePrefix != "/" {
apiPrefix = routePrefix
}

mux.Handle(
apiPrefix+"/api/v2/",
api.instrumentHandler(
Expand All @@ -190,13 +191,31 @@ func (api *API) Register(r *route.Router, routePrefix string) *http.ServeMux {
),
)

// ConnectRPC procedures are fully-qualified and already carry their own
// service version (e.g. /status.v3.StatusService/GetStatus), so mount
// them behind a version-neutral /api/ prefix. The more specific
// /api/v2/ pattern above wins via longest-prefix matching.
// RPCs are POSTs, so they are not subject to the GET concurrency limiter.
mux.Handle(
apiPrefix+"/api/",
api.instrumentHandler(
apiPrefix,
http.StripPrefix(apiPrefix+"/api", api.connect.Handler()),
),
)

return mux
}

// Update config and resolve timeout of each API. APIv2 also needs
// setAlertStatus to be updated.
func (api *API) Update(cfg *config.Config, setAlertStatus func(ctx context.Context, labels model.LabelSet)) {
api.v2.Update(cfg, setAlertStatus)
if api.v2 != nil {
api.v2.Update(cfg, setAlertStatus)
}
if api.connect != nil {
api.connect.Update(cfg)
}
}

func (api *API) limitHandler(h http.Handler) http.Handler {
Expand Down
92 changes: 92 additions & 0 deletions api/connect/connect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package apiconnect implements the experimental ConnectRPC-based
// Alertmanager API. It is always mounted alongside API v2, under the
// version-neutral /api/ prefix. ConnectRPC serves the Connect, gRPC, and
// gRPC-Web protocols from a single service definition. Each service is
// independently versioned (e.g. status.v3); the package itself carries no
// umbrella version.
package apiconnect

import (
"log/slog"
"net/http"
"sync"
"time"

"connectrpc.com/connect"
"connectrpc.com/grpchealth"
"connectrpc.com/grpcreflect"
"github.com/prometheus/common/promslog"

"github.com/prometheus/alertmanager/api/status/v3/statusv3connect"
"github.com/prometheus/alertmanager/cluster"
"github.com/prometheus/alertmanager/config"
)

// API implements the ConnectRPC service handlers for the Connect API.
type API struct {
mtx sync.RWMutex
logger *slog.Logger
peer cluster.ClusterPeer
uptime time.Time

alertmanagerConfig *config.Config
}

// NewAPI returns a new Connect API handler. Peer may be nil when clustering
// is disabled. If logger is nil, a no-op logger is used.
func NewAPI(peer cluster.ClusterPeer, logger *slog.Logger) *API {
if logger == nil {
logger = promslog.NewNopLogger()
}
return &API{
logger: logger,
peer: peer,
uptime: time.Now(),
}
}

// Update swaps in the currently loaded configuration. It is safe for
// concurrent use with the RPC handlers.
func (api *API) Update(cfg *config.Config) {
api.mtx.Lock()
defer api.mtx.Unlock()
api.alertmanagerConfig = cfg
}

// Handler returns an http.Handler serving every ConnectRPC service exposed
// by the Connect API: the versioned application services, the gRPC Health
// Checking Protocol (grpc.health.v1.Health), and gRPC server reflection
// (v1 and v1alpha, for tools such as grpcurl). Procedures are
// fully-qualified, so the returned handler is mounted at a single prefix.
func (api *API) Handler(opts ...connect.HandlerOption) http.Handler {
// serviceNames lists the fully-qualified service names advertised via
// health checking and reflection.
serviceNames := []string{
statusv3connect.StatusServiceName,
}

mux := http.NewServeMux()

mux.Handle(statusv3connect.NewStatusServiceHandler(api, opts...))

mux.Handle(grpchealth.NewHandler(grpchealth.NewStaticChecker(serviceNames...), opts...))

reflector := grpcreflect.NewStaticReflector(serviceNames...)
mux.Handle(grpcreflect.NewHandlerV1(reflector, opts...))
mux.Handle(grpcreflect.NewHandlerV1Alpha(reflector, opts...))

return mux
}
59 changes: 59 additions & 0 deletions api/connect/health_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apiconnect

import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"

"github.com/stretchr/testify/require"

"github.com/prometheus/alertmanager/api/status/v3/statusv3connect"
"github.com/prometheus/alertmanager/config"
)

// TestGRPCHealth verifies the gRPC Health Checking Protocol handler is
// mounted and reports SERVING for both the overall server ("") and the
// registered StatusService. The Health service is queried over the Connect
// protocol with JSON, which needs only an HTTP/1.1 client.
func TestGRPCHealth(t *testing.T) {
api := NewAPI(nil, nil)
api.Update(&config.Config{})

srv := httptest.NewServer(api.Handler())
t.Cleanup(srv.Close)

for _, service := range []string{"", statusv3connect.StatusServiceName} {
reqBody, err := json.Marshal(map[string]string{"service": service})
require.NoError(t, err)

resp, err := srv.Client().Post(
srv.URL+"/grpc.health.v1.Health/Check",
"application/json",
bytes.NewReader(reqBody),
)
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
require.Equal(t, http.StatusOK, resp.StatusCode)

var out struct {
Status string `json:"status"`
}
require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
require.Equal(t, "SERVING_STATUS_SERVING", out.Status)
}
}
94 changes: 94 additions & 0 deletions api/connect/status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apiconnect

import (
"context"
"sort"

"connectrpc.com/connect"
"github.com/prometheus/common/version"
"google.golang.org/protobuf/types/known/timestamppb"

statusv3 "github.com/prometheus/alertmanager/api/status/v3"
"github.com/prometheus/alertmanager/api/status/v3/statusv3connect"
)

// Ensure API satisfies the generated StatusService handler interface.
var _ statusv3connect.StatusServiceHandler = (*API)(nil)

// GetStatus returns the Alertmanager instance and cluster status.
func (api *API) GetStatus(_ context.Context, _ *connect.Request[statusv3.GetStatusRequest]) (*connect.Response[statusv3.GetStatusResponse], error) {
api.mtx.RLock()
defer api.mtx.RUnlock()

var original string
if api.alertmanagerConfig != nil {
original = api.alertmanagerConfig.String()
}

status := &statusv3.AlertmanagerStatus{
StartTime: timestamppb.New(api.uptime),
VersionInfo: &statusv3.VersionInfo{
Version: version.Version,
Revision: version.Revision,
Branch: version.Branch,
BuildUser: version.BuildUser,
BuildDate: version.BuildDate,
GoVersion: version.GoVersion,
},
Config: &statusv3.AlertmanagerConfig{
Original: original,
},
Cluster: &statusv3.ClusterStatus{
State: statusv3.ClusterStatus_STATE_DISABLED,
Peers: []*statusv3.PeerStatus{},
},
}

// If clustering is disabled, api.peer is nil and the cluster is
// reported as disabled.
if api.peer != nil {
peers := make([]*statusv3.PeerStatus, 0, len(api.peer.Peers()))
for _, n := range api.peer.Peers() {
peers = append(peers, &statusv3.PeerStatus{
Name: n.Name(),
Address: n.Address(),
})
}
sort.Slice(peers, func(i, j int) bool {
return peers[i].Name < peers[j].Name
})

status.Cluster = &statusv3.ClusterStatus{
Name: api.peer.Name(),
State: clusterState(api.peer.Status()),
Peers: peers,
}
}

return connect.NewResponse(&statusv3.GetStatusResponse{Status: status}), nil
}

// clusterState maps a cluster.ClusterPeer status string onto the proto enum.
func clusterState(s string) statusv3.ClusterStatus_State {
switch s {
case "ready":
return statusv3.ClusterStatus_STATE_READY
case "settling":
return statusv3.ClusterStatus_STATE_SETTLING
default:
return statusv3.ClusterStatus_STATE_UNSPECIFIED
}
}
Loading
Loading