Skip to content

Commit

Permalink
Add type-safe metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaelSnowden committed Jun 9, 2023
1 parent dd355b2 commit acf71a1
Show file tree
Hide file tree
Showing 15 changed files with 346 additions and 93 deletions.
65 changes: 65 additions & 0 deletions common/metrics/README.md
@@ -0,0 +1,65 @@
# Metrics for Temporal

## Glossary


### Metrics Client
A metrics client is a plain Go object that Temporal server code uses to define metrics and record individual samples.
Temporal defines its own metrics client interface called `metrics.Handler`. However, this interface has several backend
client implementations.

#### OpenTelemetry client
The OpenTelemetry client provides an interface for defining and recording metrics, called `metrics.Meter`, which
looks something like this:

```go
type Meter interface {
Int64Counter(name string, options ...Int64CounterOption) (Int64Counter, error)
}

type Int64Counter interface {
Add(ctx context.Context, incr int64, options ...AddOption)
}
```

The OTel client depends on a metrics exporter (the `metric.Reader` interface) to export metrics from the application
to a metrics backend. The only exporter that OTel currently supports is Prometheus (`prometheus.Exporter`). So, by using
OTel, you are coupling your metrics with Prometheus, at least until another exporter is available.

#### Tally client
The Tally metrics client provides similar functionality to the OTel client, but, in addition to Prometheus, it supports
StatsD, Prometheus and M3 exporter. The metrics client is provided is called `tally.Scope` and looks something like this:

```go
type Scope interface {
Counter(name string) Counter
}

type Counter interface {
Inc(delta int64)
}
```

### Time Series Database
A time series database (TSDB) is a database optimized for time-stamped or time series data like metrics. Examples
include Prometheus, InfluxDB, Graphite, and OpenTSDB.


### Metric Network Protocol
A metric protocol is a protocol for sending metric samples and definitions across a network. Examples include StatsD,
Prometheus, and Graphite.


### Metrics Gateway
A metrics gateway is a component that receives metrics from Temporal and forwards them to a time series database.
For example, a StatsD

### Metrics Reporter
A metrics reporter is a component that reports metrics from Temporal to a time series database.

#### StatsD Reporter
A StatsD rep

#### Prometheus Pull-based Reporter

#### Prometheus Pushgateway Reporter
14 changes: 7 additions & 7 deletions common/metrics/config.go
Expand Up @@ -220,7 +220,7 @@ var (
}

defaultPerUnitHistogramBoundaries = map[string][]float64{
Dimensionless: {
string(Dimensionless): {
1,
2,
5,
Expand All @@ -238,7 +238,7 @@ var (
50_000,
100_000,
},
Milliseconds: {
string(Milliseconds): {
1,
2,
5,
Expand All @@ -259,7 +259,7 @@ var (
500_000,
1_000_000, // 1000s = 16m40s
},
Bytes: {
string(Bytes): {
1024,
2048,
4096,
Expand Down Expand Up @@ -358,7 +358,7 @@ func buildTallyTimerHistogramBuckets(
return result
}

boundaries := clientConfig.PerUnitHistogramBoundaries[Milliseconds]
boundaries := clientConfig.PerUnitHistogramBoundaries[string(Milliseconds)]
result := make([]prometheus.HistogramObjective, 0, len(boundaries))
for _, boundary := range boundaries {
result = append(result, prometheus.HistogramObjective{
Expand All @@ -374,13 +374,13 @@ func setDefaultPerUnitHistogramBoundaries(clientConfig *ClientConfig) {
// In config, when overwrite default buckets, we use [dimensionless / miliseconds / bytes] as keys.
// But in code, we use [1 / ms / By] as key (to align with otel unit definition). So we do conversion here.
if bucket, ok := clientConfig.PerUnitHistogramBoundaries[UnitNameDimensionless]; ok {
buckets[Dimensionless] = bucket
buckets[string(Dimensionless)] = bucket
}
if bucket, ok := clientConfig.PerUnitHistogramBoundaries[UnitNameMilliseconds]; ok {
buckets[Milliseconds] = bucket
buckets[string(Milliseconds)] = bucket
}
if bucket, ok := clientConfig.PerUnitHistogramBoundaries[UnitNameBytes]; ok {
buckets[Bytes] = bucket
buckets[string(Bytes)] = bucket
}

clientConfig.PerUnitHistogramBoundaries = buckets
Expand Down
6 changes: 3 additions & 3 deletions common/metrics/config_test.go
Expand Up @@ -136,9 +136,9 @@ func (s *MetricsSuite) TestSetDefaultPerUnitHistogramBoundaries() {
}

customizedBoundaries := map[string][]float64{
Dimensionless: {1},
Milliseconds: defaultPerUnitHistogramBoundaries[Milliseconds],
Bytes: defaultPerUnitHistogramBoundaries[Bytes],
string(Dimensionless): {1},
string(Milliseconds): defaultPerUnitHistogramBoundaries[string(Milliseconds)],
string(Bytes): defaultPerUnitHistogramBoundaries[string(Bytes)],
}
testCases := []histogramTest{
{
Expand Down
112 changes: 82 additions & 30 deletions common/metrics/defs.go
Expand Up @@ -27,60 +27,112 @@ package metrics

// types used/defined by the package
type (
// MetricName is the name of the metric
MetricName string
// Unit is a string type that represents a unit of a metric
Unit string

MetricUnit string
// WithDescription is an option that specifies the description of a metric
WithDescription string

// metricDefinition contains the definition for a metric
metricDefinition struct {
metricName MetricName // metric name
unit MetricUnit
// withDefinition is an option that copies all the fields from a metric definition. This is unexported because
// external clients should just use the metric definition directly e.g. TimerDef.Recorder(handler).
withDefinition struct {
*MetricDef
}

// MetricDef contains the definition for a metric
MetricDef struct {
name string
unit Unit
description string
}
)

// MetricUnit supported values
// Unit supported values
// Values are pulled from https://pkg.go.dev/golang.org/x/exp/event#Unit
const (
Dimensionless = "1"
Milliseconds = "ms"
Bytes = "By"
Dimensionless Unit = "1"
Milliseconds Unit = "ms"
Bytes Unit = "By"
)

// Empty returns true if the metricName is an empty string
func (mn MetricName) Empty() bool {
return mn == ""
func (unit Unit) apply(definition *MetricDef) *MetricDef {
definition.unit = unit
return definition
}

func (description WithDescription) apply(definition *MetricDef) *MetricDef {
definition.description = string(description)
return definition
}

// String returns string representation of this metric name
func (mn MetricName) String() string {
return string(mn)
func (definition withDefinition) apply(_ *MetricDef) *MetricDef {
return definition.MetricDef
}

func (md metricDefinition) GetMetricName() string {
return md.metricName.String()
func (md *MetricDef) GetMetricName() string {
return md.name
}

func (md metricDefinition) GetMetricUnit() MetricUnit {
func (md *MetricDef) GetMetricUnit() Unit {
return md.unit
}

func NewTimerDef(name string) metricDefinition {
return metricDefinition{metricName: MetricName(name), unit: Milliseconds}
type TimerDef struct {
MetricDef
}

func (td TimerDef) Recorder(handler Handler) TimerIface {
return handler.Timer(td.name, withDefinition{&td.MetricDef})
}

func newMetricDef(name string, opts []Option) MetricDef {
d := &MetricDef{name: name}
for _, opt := range opts {
d = opt.apply(d)
}
return *d
}

func NewTimerDef(name string, opts ...Option) TimerDef {
return TimerDef{newMetricDef(name, append(opts, Milliseconds))}
}

type HistogramDef struct {
MetricDef
}

func (hd *HistogramDef) Recorder(handler Handler) HistogramIface {
return handler.Histogram(hd.name, hd.unit, withDefinition{&hd.MetricDef})
}

func NewBytesHistogramDef(name string, opts ...Option) HistogramDef {
return HistogramDef{newMetricDef(name, append(opts, Bytes))}
}

func NewDimensionlessHistogramDef(name string, opts ...Option) HistogramDef {
return HistogramDef{newMetricDef(name, append(opts, Dimensionless))}
}

type CounterDef struct {
MetricDef
}

func NewCounterDef(name string, opts ...Option) CounterDef {
return CounterDef{newMetricDef(name, opts)}
}

func NewBytesHistogramDef(name string) metricDefinition {
return metricDefinition{metricName: MetricName(name), unit: Bytes}
func (cd *CounterDef) Recorder(handler Handler) CounterIface {
return handler.Counter(cd.name, withDefinition{&cd.MetricDef})
}

func NewDimensionlessHistogramDef(name string) metricDefinition {
return metricDefinition{metricName: MetricName(name), unit: Dimensionless}
type GaugeDef struct {
MetricDef
}

func NewCounterDef(name string) metricDefinition {
return metricDefinition{metricName: MetricName(name)}
func (gd *GaugeDef) Recorder(handler Handler) GaugeIface {
return handler.Gauge(gd.name)
}

func NewGaugeDef(name string) metricDefinition {
return metricDefinition{metricName: MetricName(name)}
func NewGaugeDef(name string, opts ...Option) GaugeDef {
return GaugeDef{newMetricDef(name, opts)}
}
12 changes: 8 additions & 4 deletions common/metrics/metrics.go
Expand Up @@ -43,16 +43,16 @@ type (
WithTags(...Tag) Handler

// Counter obtains a counter for the given name and MetricOptions.
Counter(string) CounterIface
Counter(string, ...Option) CounterIface

// Gauge obtains a gauge for the given name and MetricOptions.
Gauge(string) GaugeIface
Gauge(string, ...Option) GaugeIface

// Timer obtains a timer for the given name and MetricOptions.
Timer(string) TimerIface
Timer(string, ...Option) TimerIface

// Histogram obtains a histogram for the given name and MetricOptions.
Histogram(string, MetricUnit) HistogramIface
Histogram(string, Unit, ...Option) HistogramIface

Stop(log.Logger)
}
Expand Down Expand Up @@ -88,6 +88,10 @@ type (
GaugeFunc func(float64, ...Tag)
TimerFunc func(time.Duration, ...Tag)
HistogramFunc func(int64, ...Tag)

Option interface {
apply(*MetricDef) *MetricDef
}
)

func (c CounterFunc) Record(v int64, tags ...Tag) { c(v, tags...) }
Expand Down

0 comments on commit acf71a1

Please sign in to comment.