Support Prometheus metrics as default for all components (#516)

cmd/agent/app/builder.go

@@ -154,8 +154,8 @@ func (b *Builder) CreateAgent(logger *zap.Logger) (*Agent, error) {
 		return nil, err
 	}
 	httpServer := b.HTTPServer.GetHTTPServer(b.CollectorServiceName, mainReporter.Channel(), mFactory)
-	if b.metricsFactory == nil {
-		b.Metrics.RegisterHandler(httpServer.Handler.(*http.ServeMux))
+	if h := b.Metrics.Handler(); b.metricsFactory != nil && h != nil {
+		httpServer.Handler.(*http.ServeMux).Handle(b.Metrics.HTTPRoute, h)
 	}
 	return NewAgent(processors, httpServer, logger), nil
 }

cmd/agent/app/builder_test.go

@@ -126,6 +126,19 @@ func TestBuilderMetrics(t *testing.T) {
 	assert.Equal(t, mf, mf2)
 }
 
+func TestBuilderMetricsHandler(t *testing.T) {
+	b := &Builder{}
+	b.Metrics.Backend = "expvar"
+	b.Metrics.HTTPRoute = "/expvar"
+	factory, err := b.Metrics.CreateMetricsFactory("test")
+	assert.NoError(t, err)
+	assert.NotNil(t, factory)
+	b.metricsFactory = factory
+	agent, err := b.CreateAgent(zap.NewNop())
+	assert.NoError(t, err)
+	assert.NotNil(t, agent)
+}
+
 func TestBuilderMetricsError(t *testing.T) {
 	b := &Builder{}
 	b.Metrics.Backend = "invalid"

cmd/agent/app/httpserver/server.go

@@ -59,25 +59,25 @@ type httpHandler struct {
 	manager ClientConfigManager
 	metrics struct {
 		// Number of good sampling requests
-		SamplingRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"result=ok,type=sampling"`
+		SamplingRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"type=sampling"`
 
 		// Number of good sampling requests against the old endpoint / using Thrift 0.9.2 enum codes
-		LegacySamplingRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"result=ok,type=sampling-legacy"`
+		LegacySamplingRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"type=sampling-legacy"`
 
 		// Number of good baggage requests
-		BaggageRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"result=ok,type=baggage"`
+		BaggageRequestSuccess metrics.Counter `metric:"http-server.requests" tags:"type=baggage"`
 
 		// Number of bad requests (400s)
-		BadRequest metrics.Counter `metric:"http-server.requests" tags:"result=err,status=4xx"`
+		BadRequest metrics.Counter `metric:"http-server.errors" tags:"status=4xx,source=all"`
 
 		// Number of collector proxy failures
-		TCollectorProxyFailures metrics.Counter `metric:"http-server.requests" tags:"result=err,status=5xx,type=tcollector-proxy"`
+		TCollectorProxyFailures metrics.Counter `metric:"http-server.errors" tags:"status=5xx,source=tcollector-proxy"`
 
 		// Number of bad responses due to malformed thrift
-		BadThriftFailures metrics.Counter `metric:"http-server.requests" tags:"result=err,status=5xx,type=thrift"`
+		BadThriftFailures metrics.Counter `metric:"http-server.errors" tags:"status=5xx,source=thrift"`
 
 		// Number of failed response writes from http server
-		WriteFailures metrics.Counter `metric:"http-server.requests" tags:"result=err,status=5xx,type=write"`
+		WriteFailures metrics.Counter `metric:"http-server.errors" tags:"status=5xx,source=write"`
 	}
 }
 

cmd/agent/app/httpserver/server_test.go

@@ -101,9 +101,9 @@ func TestHTTPHandler(t *testing.T) {
 
 		// handler must emit metrics
 		testutils.AssertCounterMetrics(t, ts.metricsFactory, []testutils.ExpectedMetric{
-			{Name: "http-server.requests", Tags: map[string]string{"result": "ok", "type": "sampling"}, Value: 1},
-			{Name: "http-server.requests", Tags: map[string]string{"result": "ok", "type": "sampling-legacy"}, Value: 1},
-			{Name: "http-server.requests", Tags: map[string]string{"result": "ok", "type": "baggage"}, Value: 1},
+			{Name: "http-server.requests", Tags: map[string]string{"type": "sampling"}, Value: 1},
+			{Name: "http-server.requests", Tags: map[string]string{"type": "sampling-legacy"}, Value: 1},
+			{Name: "http-server.requests", Tags: map[string]string{"type": "baggage"}, Value: 1},
 		}...)
 	})
 }
@@ -124,7 +124,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:  http.StatusBadRequest,
 			body:        "'service' parameter must be provided once\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "4xx"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "all", "status": "4xx"}, Value: 1},
 			},
 		},
 		{
@@ -133,7 +133,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:  http.StatusBadRequest,
 			body:        "'service' parameter must be provided once\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "4xx"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "all", "status": "4xx"}, Value: 1},
 			},
 		},
 		{
@@ -142,7 +142,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:  http.StatusBadRequest,
 			body:        "'service' parameter must be provided once\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "4xx"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "all", "status": "4xx"}, Value: 1},
 			},
 		},
 		{
@@ -151,7 +151,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:  http.StatusInternalServerError,
 			body:        "tcollector error: no mock response provided\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "5xx", "type": "tcollector-proxy"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "tcollector-proxy", "status": "5xx"}, Value: 1},
 			},
 		},
 		{
@@ -160,7 +160,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:  http.StatusInternalServerError,
 			body:        "tcollector error: no mock response provided\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "5xx", "type": "tcollector-proxy"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "tcollector-proxy", "status": "5xx"}, Value: 1},
 			},
 		},
 		{
@@ -170,7 +170,7 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			statusCode:           http.StatusInternalServerError,
 			body:                 "Cannot marshall Thrift to JSON\n",
 			metrics: []mTestutils.ExpectedMetric{
-				{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "5xx", "type": "thrift"}, Value: 1},
+				{Name: "http-server.errors", Tags: map[string]string{"source": "thrift", "status": "5xx"}, Value: 1},
 			},
 		},
 	}
@@ -203,13 +203,13 @@ func TestHTTPHandlerErrors(t *testing.T) {
 			handler.serveSamplingHTTP(w, req, false)
 
 			mTestutils.AssertCounterMetrics(t, ts.metricsFactory,
-				mTestutils.ExpectedMetric{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "5xx", "type": "write"}, Value: 1})
+				mTestutils.ExpectedMetric{Name: "http-server.errors", Tags: map[string]string{"source": "write", "status": "5xx"}, Value: 1})
 
 			req = httptest.NewRequest("GET", "http://localhost:80/baggageRestrictions?service=X", nil)
 			handler.serveBaggageHTTP(w, req)
 
 			mTestutils.AssertCounterMetrics(t, ts.metricsFactory,
-				mTestutils.ExpectedMetric{Name: "http-server.requests", Tags: map[string]string{"result": "err", "status": "5xx", "type": "write"}, Value: 2})
+				mTestutils.ExpectedMetric{Name: "http-server.errors", Tags: map[string]string{"source": "write", "status": "5xx"}, Value: 2})
 		})
 	})
 }

cmd/collector/app/builder/span_handler_builder.go

@@ -116,7 +116,7 @@ func (spanHb *SpanHandlerBuilder) initElasticStore(esBuilder escfg.ClientBuilder
 // BuildHandlers builds span handlers (Zipkin, Jaeger)
 func (spanHb *SpanHandlerBuilder) BuildHandlers() (app.ZipkinSpansHandler, app.JaegerBatchesHandler) {
 	hostname, _ := os.Hostname()
-	hostMetrics := spanHb.metricsFactory.Namespace(hostname, nil)
+	hostMetrics := spanHb.metricsFactory.Namespace("", map[string]string{"host": hostname})
 
 	zSanitizer := zs.NewChainedSanitizer(
 		zs.NewSpanDurationSanitizer(),

cmd/collector/app/zipkin/json.go

@@ -240,8 +240,6 @@ func binAnnoToThrift(ba binaryAnnotation) (*zipkincore.BinaryAnnotation, error)
 	default:
 		str := fmt.Sprintf("%s", ba.Value)
 		val = []byte(str)
-		fmt.Println("default")
-		fmt.Println(str)
 		valType = zipkincore.AnnotationType_STRING
 	}
 

cmd/collector/main.go

@@ -26,8 +26,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/uber/jaeger-lib/metrics/go-kit"
-	"github.com/uber/jaeger-lib/metrics/go-kit/expvar"
 	"github.com/uber/tchannel-go"
 	"github.com/uber/tchannel-go/thrift"
 	"go.uber.org/zap"
@@ -41,6 +39,7 @@ import (
 	esFlags "github.com/jaegertracing/jaeger/cmd/flags/es"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/healthcheck"
+	pMetrics "github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/pkg/recoveryhandler"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	jc "github.com/jaegertracing/jaeger/thrift-gen/jaeger"
@@ -75,11 +74,14 @@ func main() {
 
 			casOptions.InitFromViper(v)
 			esOptions.InitFromViper(v)
-
-			baseMetrics := xkit.Wrap(serviceName, expvar.NewFactory(10))
-
+			mBldr := new(pMetrics.Builder).InitFromViper(v)
 			builderOpts := new(builder.CollectorOptions).InitFromViper(v)
 
+			metricsFactory, err := mBldr.CreateMetricsFactory("jaeger-collector")
+			if err != nil {
+				logger.Fatal("Cannot create metrics factory.", zap.Error(err))
+			}
+
 			hc, err := healthcheck.Serve(http.StatusServiceUnavailable, builderOpts.CollectorHealthCheckHTTPPort, logger)
 			if err != nil {
 				logger.Fatal("Could not start the health check server.", zap.Error(err))
@@ -91,7 +93,7 @@ func main() {
 				basicB.Options.CassandraSessionOption(casOptions.GetPrimary()),
 				basicB.Options.ElasticClientOption(esOptions.GetPrimary()),
 				basicB.Options.LoggerOption(logger),
-				basicB.Options.MetricsFactoryOption(baseMetrics),
+				basicB.Options.MetricsFactoryOption(metricsFactory),
 			)
 			if err != nil {
 				logger.Fatal("Unable to set up builder", zap.Error(err))
@@ -116,6 +118,10 @@ func main() {
 			r := mux.NewRouter()
 			apiHandler := app.NewAPIHandler(jaegerBatchesHandler)
 			apiHandler.RegisterRoutes(r)
+			if h := mBldr.Handler(); h != nil {
+				logger.Info("Registering metrics handler with HTTP server", zap.String("route", mBldr.HTTPRoute))
+				r.Handle(mBldr.HTTPRoute, h)
+			}
 			httpPortStr := ":" + strconv.Itoa(builderOpts.CollectorHTTPPort)
 			recoveryHandler := recoveryhandler.NewRecoveryHandler(logger, true)
 
@@ -149,6 +155,7 @@ func main() {
 		builder.AddFlags,
 		casOptions.AddFlags,
 		esOptions.AddFlags,
+		pMetrics.AddFlags,
 	)
 
 	if error := command.Execute(); error != nil {

cmd/query/main.go

@@ -27,8 +27,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	jaegerClientConfig "github.com/uber/jaeger-client-go/config"
-	"github.com/uber/jaeger-lib/metrics/go-kit"
-	"github.com/uber/jaeger-lib/metrics/go-kit/expvar"
 	"go.uber.org/zap"
 
 	basicB "github.com/jaegertracing/jaeger/cmd/builder"
@@ -39,6 +37,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/query/app/builder"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/healthcheck"
+	pMetrics "github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/pkg/recoveryhandler"
 	"github.com/jaegertracing/jaeger/pkg/version"
 )
@@ -70,13 +69,17 @@ func main() {
 			casOptions.InitFromViper(v)
 			esOptions.InitFromViper(v)
 			queryOpts := new(builder.QueryOptions).InitFromViper(v)
+			mBldr := new(pMetrics.Builder).InitFromViper(v)
 
 			hc, err := healthcheck.Serve(http.StatusServiceUnavailable, queryOpts.HealthCheckHTTPPort, logger)
 			if err != nil {
 				logger.Fatal("Could not start the health check server.", zap.Error(err))
 			}
 
-			metricsFactory := xkit.Wrap("jaeger-query", expvar.NewFactory(10))
+			metricsFactory, err := mBldr.CreateMetricsFactory("jaeger-query")
+			if err != nil {
+				logger.Fatal("Cannot create metrics factory.", zap.Error(err))
+			}
 
 			tracer, closer, err := jaegerClientConfig.Configuration{
 				Sampler: &jaegerClientConfig.SamplerConfig{
@@ -111,6 +114,12 @@ func main() {
 			r := mux.NewRouter()
 			apiHandler.RegisterRoutes(r)
 			registerStaticHandler(r, logger, queryOpts)
+
+			if h := mBldr.Handler(); h != nil {
+				logger.Info("Registering metrics handler with HTTP server", zap.String("route", mBldr.HTTPRoute))
+				r.Handle(mBldr.HTTPRoute, h)
+			}
+
 			portStr := ":" + strconv.Itoa(queryOpts.Port)
 			compressHandler := handlers.CompressHandler(r)
 			recoveryHandler := recoveryhandler.NewRecoveryHandler(logger, true)
@@ -142,6 +151,7 @@ func main() {
 		flags.AddFlags,
 		casOptions.AddFlags,
 		esOptions.AddFlags,
+		pMetrics.AddFlags,
 		builder.AddFlags,
 	)
 

cmd/standalone/main.go

@@ -23,12 +23,11 @@ import (
 	"strconv"
 
 	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	jaegerClientConfig "github.com/uber/jaeger-client-go/config"
 	"github.com/uber/jaeger-lib/metrics"
-	"github.com/uber/jaeger-lib/metrics/go-kit"
-	"github.com/uber/jaeger-lib/metrics/go-kit/expvar"
 	"github.com/uber/tchannel-go"
 	"github.com/uber/tchannel-go/thrift"
 	"go.uber.org/zap"
@@ -73,15 +72,19 @@ func main() {
 			runtime.GOMAXPROCS(runtime.NumCPU())
 			cOpts := new(collector.CollectorOptions).InitFromViper(v)
 			qOpts := new(query.QueryOptions).InitFromViper(v)
+			mBldr := new(pMetrics.Builder).InitFromViper(v)
 
-			metricsFactory := xkit.Wrap("jaeger-standalone", expvar.NewFactory(10))
+			metricsFactory, err := mBldr.CreateMetricsFactory("jaeger-standalone")
+			if err != nil {
+				return errors.Wrap(err, "Cannot create metrics factory")
+			}
 			memStore := memory.NewStore()
 
 			builder := &agentApp.Builder{}
 			builder.InitFromViper(v)
 			startAgent(builder, cOpts, logger, metricsFactory)
 			startCollector(cOpts, sFlags, logger, metricsFactory, memStore)
-			startQuery(qOpts, sFlags, logger, metricsFactory, memStore)
+			startQuery(qOpts, sFlags, logger, metricsFactory, mBldr, memStore)
 			select {}
 		},
 	}
@@ -202,6 +205,7 @@ func startQuery(
 	sFlags *flags.SharedFlags,
 	logger *zap.Logger,
 	baseFactory metrics.Factory,
+	metricsBuilder *pMetrics.Builder,
 	memoryStore *memory.Store,
 ) {
 	metricsFactory := baseFactory.Namespace("jaeger-query", nil)
@@ -233,9 +237,16 @@ func startQuery(
 		queryApp.HandlerOptions.Prefix(qOpts.Prefix),
 		queryApp.HandlerOptions.Logger(logger),
 		queryApp.HandlerOptions.Tracer(tracer))
+
 	r := mux.NewRouter()
 	apiHandler.RegisterRoutes(r)
 	registerStaticHandler(r, logger, qOpts)
+
+	if h := metricsBuilder.Handler(); h != nil {
+		logger.Info("Registering metrics handler with jaeger-query HTTP server", zap.String("route", metricsBuilder.HTTPRoute))
+		r.Handle(metricsBuilder.HTTPRoute, h)
+	}
+
 	portStr := ":" + strconv.Itoa(qOpts.Port)
 	recoveryHandler := recoveryhandler.NewRecoveryHandler(logger, true)
 	logger.Info("Starting jaeger-query HTTP server", zap.Int("port", qOpts.Port))