Additional metrics for New Relic

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Add `IMGPROXY_PREFERRED_FORMATS` config.
 - Add `IMGPROXY_REQUESTS_QUEUE_SIZE` config.
 - Add sending additional metrics to Datadog and `IMGPROXY_DATADOG_ENABLE_ADDITIONAL_METRICS` config.
+- Add sending additional metrics to New Relic.
 
 ### Change
 - Change `IMGPROXY_MAX_CLIENTS` default value to 2048.

docs/new_relic.md

@@ -14,3 +14,12 @@ imgproxy will send the following info to New Relic:
 * Image downloading time
 * Image processing time
 * Errors that occurred while downloading and processing an image
+
+Additionally, imgproxy sends the following metrics over [Metrics API](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/):
+
+* `imgproxy.buffer.size`: a summary of the download/gzip buffers sizes (in bytes)
+* `imgproxy.buffer.default_size`: calibrated default buffer size (in bytes)
+* `imgproxy.buffer.max_size`: calibrated maximum buffer size (in bytes)
+* `imgproxy.vips.memory`: libvips memory usage (in bytes)
+* `imgproxy.vips.max_memory`: libvips maximum memory usage (in bytes)
+* `imgproxy.vips.allocs`: the number of active vips allocations

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/matoous/go-nanoid/v2 v2.0.0
 	github.com/ncw/swift/v2 v2.0.1
 	github.com/newrelic/go-agent/v3 v3.16.1
+	github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -884,6 +884,8 @@ github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
 github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
 github.com/newrelic/go-agent/v3 v3.16.1 h1:gH053irA4rIAySGSvMc2grKiKNhrM4gCzc+p3M+rqAE=
 github.com/newrelic/go-agent/v3 v3.16.1/go.mod h1:BFJOlbZWRlPTXKYIC1TTTtQKTnYntEJaU0VU507hDc0=
+github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1 h1:6OX5VXMuj2salqNBc41eXKz6K+nV6OB/hhlGnAKCbwU=
+github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1/go.mod h1:2kY6OeOxrJ+RIQlVjWDc/pZlT3MIf30prs6drzMfJ6E=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=

metrics/metrics.go

@@ -22,6 +22,7 @@ func Init() error {
 }
 
 func Stop() {
+	newrelic.Stop()
 	datadog.Stop()
 }
 
@@ -81,15 +82,18 @@ func SendError(ctx context.Context, errType string, err error) {
 
 func ObserveBufferSize(t string, size int) {
 	prometheus.ObserveBufferSize(t, size)
+	newrelic.ObserveBufferSize(t, size)
 	datadog.ObserveBufferSize(t, size)
 }
 
 func SetBufferDefaultSize(t string, size int) {
 	prometheus.SetBufferDefaultSize(t, size)
+	newrelic.SetBufferDefaultSize(t, size)
 	datadog.SetBufferDefaultSize(t, size)
 }
 
 func SetBufferMaxSize(t string, size int) {
 	prometheus.SetBufferMaxSize(t, size)
+	newrelic.SetBufferMaxSize(t, size)
 	datadog.SetBufferMaxSize(t, size)
 }

metrics/newrelic/newrelic.go

@@ -3,19 +3,48 @@ package newrelic
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/newrelic/go-agent/v3/newrelic"
+	"github.com/newrelic/newrelic-telemetry-sdk-go/telemetry"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/imgproxy/imgproxy/v3/config"
 	"github.com/imgproxy/imgproxy/v3/metrics/errformat"
-	"github.com/newrelic/go-agent/v3/newrelic"
 )
 
 type transactionCtxKey struct{}
 
+type GaugeFunc func() float64
+
+const (
+	defaultMetricURL = "https://metric-api.newrelic.com/metric/v1"
+	euMetricURL      = "https://metric-api.eu.newrelic.com/metric/v1"
+)
+
 var (
-	enabled = false
+	enabled          = false
+	enabledHarvester = false
+
+	app       *newrelic.Application
+	harvester *telemetry.Harvester
+
+	harvesterCtx       context.Context
+	harvesterCtxCancel context.CancelFunc
+
+	gaugeFuncs      = make(map[string]GaugeFunc)
+	gaugeFuncsMutex sync.RWMutex
+
+	bufferSummaries      = make(map[string]*telemetry.Summary)
+	bufferSummariesMutex sync.RWMutex
+
+	interval = 10 * time.Second
 
-	newRelicApp *newrelic.Application
+	licenseEuRegex = regexp.MustCompile(`(^eu.+?)x`)
 )
 
 func Init() error {
@@ -30,7 +59,7 @@ func Init() error {
 
 	var err error
 
-	newRelicApp, err = newrelic.NewApplication(
+	app, err = newrelic.NewApplication(
 		newrelic.ConfigAppName(name),
 		newrelic.ConfigLicense(config.NewRelicKey),
 		func(c *newrelic.Config) {
@@ -44,11 +73,47 @@ func Init() error {
 		return fmt.Errorf("Can't init New Relic agent: %s", err)
 	}
 
+	harvesterAttributes := map[string]interface{}{"appName": name}
+	for k, v := range config.NewRelicLabels {
+		harvesterAttributes[k] = v
+	}
+
+	metricsURL := defaultMetricURL
+	if licenseEuRegex.MatchString(config.NewRelicKey) {
+		metricsURL = euMetricURL
+	}
+
+	harvester, err = telemetry.NewHarvester(
+		telemetry.ConfigAPIKey(config.NewRelicKey),
+		telemetry.ConfigCommonAttributes(harvesterAttributes),
+		telemetry.ConfigHarvestPeriod(0), // Don't harvest automatically
+		telemetry.ConfigMetricsURLOverride(metricsURL),
+		telemetry.ConfigBasicErrorLogger(log.StandardLogger().WithField("from", "newrelic").WriterLevel(log.WarnLevel)),
+	)
+	if err == nil {
+		harvesterCtx, harvesterCtxCancel = context.WithCancel(context.Background())
+		enabledHarvester = true
+		go runMetricsCollector()
+	} else {
+		log.Warnf("Can't init New Relic telemetry harvester: %s", err)
+	}
+
 	enabled = true
 
 	return nil
 }
 
+func Stop() {
+	if enabled {
+		app.Shutdown(5 * time.Second)
+
+		if enabledHarvester {
+			harvesterCtxCancel()
+			harvester.HarvestNow(context.Background())
+		}
+	}
+}
+
 func Enabled() bool {
 	return enabled
 }
@@ -58,7 +123,7 @@ func StartTransaction(ctx context.Context, rw http.ResponseWriter, r *http.Reque
 		return ctx, func() {}, rw
 	}
 
-	txn := newRelicApp.StartTransaction("request")
+	txn := app.StartTransaction("request")
 	txn.SetWebRequestHTTP(r)
 	newRw := txn.SetWebResponse(rw)
 	cancel := func() { txn.End() }
@@ -90,3 +155,100 @@ func SendError(ctx context.Context, errType string, err error) {
 		})
 	}
 }
+
+func AddGaugeFunc(name string, f GaugeFunc) {
+	gaugeFuncsMutex.Lock()
+	defer gaugeFuncsMutex.Unlock()
+
+	gaugeFuncs["imgproxy."+name] = f
+}
+
+func ObserveBufferSize(t string, size int) {
+	if enabledHarvester {
+		bufferSummariesMutex.Lock()
+		defer bufferSummariesMutex.Unlock()
+
+		summary, ok := bufferSummaries[t]
+		if !ok {
+			summary = &telemetry.Summary{
+				Name:       "imgproxy.buffer.size",
+				Attributes: map[string]interface{}{"buffer_type": t},
+				Timestamp:  time.Now(),
+			}
+			bufferSummaries[t] = summary
+		}
+
+		sizef := float64(size)
+
+		summary.Count += 1
+		summary.Sum += sizef
+		summary.Min = math.Min(summary.Min, sizef)
+		summary.Max = math.Max(summary.Max, sizef)
+	}
+}
+
+func SetBufferDefaultSize(t string, size int) {
+	if enabledHarvester {
+		harvester.RecordMetric(telemetry.Gauge{
+			Name:       "imgproxy.buffer.default_size",
+			Value:      float64(size),
+			Attributes: map[string]interface{}{"buffer_type": t},
+			Timestamp:  time.Now(),
+		})
+	}
+}
+
+func SetBufferMaxSize(t string, size int) {
+	if enabledHarvester {
+		harvester.RecordMetric(telemetry.Gauge{
+			Name:       "imgproxy.buffer.max_size",
+			Value:      float64(size),
+			Attributes: map[string]interface{}{"buffer_type": t},
+			Timestamp:  time.Now(),
+		})
+	}
+}
+
+func runMetricsCollector() {
+	tick := time.NewTicker(interval)
+	defer tick.Stop()
+	for {
+		select {
+		case <-tick.C:
+			func() {
+				gaugeFuncsMutex.RLock()
+				defer gaugeFuncsMutex.RUnlock()
+
+				for name, f := range gaugeFuncs {
+					harvester.RecordMetric(telemetry.Gauge{
+						Name:      name,
+						Value:     f(),
+						Timestamp: time.Now(),
+					})
+				}
+			}()
+
+			func() {
+				bufferSummariesMutex.RLock()
+				defer bufferSummariesMutex.RUnlock()
+
+				now := time.Now()
+
+				for _, summary := range bufferSummaries {
+					summary.Interval = now.Sub(summary.Timestamp)
+					harvester.RecordMetric(*summary)
+
+					summary.Timestamp = now
+					summary.Count = 0
+					summary.Sum = 0
+					summary.Min = 0
+					summary.Max = 0
+				}
+			}()
+
+			harvester.HarvestNow(harvesterCtx)
+		case <-harvesterCtx.Done():
+			return
+		}
+	}
+}

vips/vips.go

@@ -22,6 +22,7 @@ import (
 	"github.com/imgproxy/imgproxy/v3/imagedata"
 	"github.com/imgproxy/imgproxy/v3/imagetype"
 	"github.com/imgproxy/imgproxy/v3/metrics/datadog"
+	"github.com/imgproxy/imgproxy/v3/metrics/newrelic"
 	"github.com/imgproxy/imgproxy/v3/metrics/prometheus"
 )
 
@@ -100,6 +101,10 @@ func Init() error {
 	datadog.AddGaugeFunc("vips.max_memory", GetMemHighwater)
 	datadog.AddGaugeFunc("vips.allocs", GetAllocs)
 
+	newrelic.AddGaugeFunc("vips.memory", GetMem)
+	newrelic.AddGaugeFunc("vips.max_memory", GetMemHighwater)
+	newrelic.AddGaugeFunc("vips.allocs", GetAllocs)
+
 	return nil
 }