Always report service error as additional metric with error-type tag (#…

…2741)
temporalio · Apr 19, 2022 · 762bf91 · 762bf91
1 parent 6deeed5
commit 762bf91
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 0 deletions.
diff --git a/common/metrics/defs.go b/common/metrics/defs.go
@@ -1754,6 +1754,7 @@ const (
 	ServiceRequests = iota
 	ServicePendingRequests
 	ServiceFailures
+	ServiceFailuresWithType
 	ServiceCriticalFailures
 	ServiceLatency
 	ServiceLatencyNoUserLatency
@@ -2253,6 +2254,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		ServiceRequests:                                     NewCounterDef("service_requests"),
 		ServicePendingRequests:                              NewGaugeDef("service_pending_requests"),
 		ServiceFailures:                                     NewCounterDef("service_errors"),
+		ServiceFailuresWithType:                             NewCounterDef("service_errors_with_type"),
 		ServiceCriticalFailures:                             NewCounterDef("service_errors_critical"),
 		ServiceLatency:                                      NewTimerDef("service_latency"),
 		ServiceLatencyNoUserLatency:                         NewTimerDef("service_latency_nouserlatency"),

diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go
@@ -147,6 +147,8 @@ func (ti *TelemetryInterceptor) handleError(
 	err error,
 ) {
 
+	scope.Tagged(metrics.ServiceErrorTypeTag(err)).IncCounter(metrics.ServiceFailuresWithType)
+
 	if common.IsContextDeadlineExceededErr(err) {
 		scope.IncCounter(metrics.ServiceErrContextTimeoutCounter)
 		return

diff --git a/service/frontend/adminHandler.go b/service/frontend/adminHandler.go
@@ -1628,6 +1628,8 @@ func (adh *AdminHandler) startRequestProfile(scope int) (metrics.Scope, metrics.
 }
 
 func (adh *AdminHandler) error(err error, scope metrics.Scope) error {
+	scope.Tagged(metrics.ServiceErrorTypeTag(err)).IncCounter(metrics.ServiceFailuresWithType)
+
 	switch err := err.(type) {
 	case *serviceerror.Unavailable:
 		adh.logger.Error("unavailable error", tag.Error(err))

diff --git a/service/frontend/operator_handler.go b/service/frontend/operator_handler.go
@@ -346,6 +346,8 @@ func (h *OperatorHandlerImpl) startRequestProfile(scope int) (metrics.Scope, met
 }
 
 func (h *OperatorHandlerImpl) error(err error, scope metrics.Scope, endpointName string) error {
+	scope.Tagged(metrics.ServiceErrorTypeTag(err)).IncCounter(metrics.ServiceFailuresWithType)
+
 	switch err := err.(type) {
 	case *serviceerror.Unavailable:
 		h.logger.Error("Unavailable error.", tag.Error(err), tag.Endpoint(endpointName))