Add metrics API endpoint and command to retrive server telemetry.

In addition to sending telemetry to an endpoint, this commit now
allows metrics to be viewed by calling the Sherpa API. A command
can also be used, which wraps the API, to output the telemetry
information to the terminal. The API and command are nested under
the system commands and endpoints.

cmd/system/command.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/jrasell/sherpa/cmd/system/health"
 	"github.com/jrasell/sherpa/cmd/system/info"
+	"github.com/jrasell/sherpa/cmd/system/metrics"
 	"github.com/sean-/sysexits"
 	"github.com/spf13/cobra"
 )
@@ -37,5 +38,9 @@ func registerCommands(rootCmd *cobra.Command) error {
 		return err
 	}
 
+	if err := metrics.RegisterCommand(rootCmd); err != nil {
+		return err
+	}
+
 	return health.RegisterCommand(rootCmd)
 }

cmd/system/metrics/command.go

@@ -0,0 +1,76 @@
+package metrics
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/jrasell/sherpa/pkg/api"
+	clientCfg "github.com/jrasell/sherpa/pkg/config/client"
+	"github.com/ryanuber/columnize"
+	"github.com/sean-/sysexits"
+	"github.com/spf13/cobra"
+)
+
+const (
+	outputHeader = "Name|Type|Value"
+)
+
+func RegisterCommand(rootCmd *cobra.Command) error {
+	cmd := &cobra.Command{
+		Use:   "metrics",
+		Short: "Retrieve metrics from a Sherpa server",
+		Run: func(cmd *cobra.Command, args []string) {
+			runInfo(cmd, args)
+		},
+	}
+	rootCmd.AddCommand(cmd)
+
+	return nil
+}
+
+func runInfo(_ *cobra.Command, _ []string) {
+	clientConfig := clientCfg.GetConfig()
+	mergedConfig := api.DefaultConfig(&clientConfig)
+
+	client, err := api.NewClient(mergedConfig)
+	if err != nil {
+		fmt.Println("Error setting up Sherpa client:", err)
+		os.Exit(sysexits.Software)
+	}
+
+	metrics, err := client.System().Metrics()
+	if err != nil {
+		fmt.Println("Error calling server metrics:", err)
+		os.Exit(sysexits.Software)
+	}
+
+	out := []string{outputHeader}
+
+	for i := range metrics.Gauges {
+		out = append(out, fmt.Sprintf("%s|%s|%v",
+			metrics.Gauges[i].Name, "Gauge", metrics.Gauges[i].Value))
+	}
+
+	for i := range metrics.Counters {
+		out = append(out, fmt.Sprintf("%s|%s|%v",
+			metrics.Counters[i].Name, "Counter", metrics.Counters[i].Mean))
+	}
+
+	for i := range metrics.Samples {
+		out = append(out, fmt.Sprintf("%s|%s|%v",
+			metrics.Samples[i].Name, "Counter", metrics.Samples[i].Mean))
+	}
+
+	// If there are no metrics to print (happens during initial server startup)
+	// then we don't want to just print the header so perform a check so the
+	// CLI is nice and tidy.
+	if len(out) > 1 {
+		fmt.Println(formatList(out))
+	}
+}
+
+func formatList(in []string) string {
+	columnConf := columnize.DefaultConfig()
+	columnConf.Empty = "<none>"
+	return columnize.Format(in, columnConf)
+}

go.mod

@@ -3,39 +3,26 @@ module github.com/jrasell/sherpa
 go 1.12
 
 require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
-	github.com/beorn7/perks v1.0.0 // indirect
-	github.com/davecgh/go-spew v1.1.1
-	github.com/go-logfmt/logfmt v0.4.0 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
 	github.com/gorilla/mux v1.7.1
 	github.com/hashicorp/consul/api v1.1.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/go-rootcerts v1.0.0
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/hashicorp/nomad/api v0.0.0-20190508234936-7ba2378a159e
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/kisielk/errcheck v1.2.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/mattn/go-isatty v0.0.7
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
-	github.com/prometheus/common v0.4.0 // indirect
-	github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 // indirect
 	github.com/rs/zerolog v1.14.3
 	github.com/ryanuber/columnize v2.1.0+incompatible
 	github.com/sean-/sysexits v0.0.0-20171026162210-598690305aaa
-	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.3.2
-	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
 	golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
 	golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f // indirect
 	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/tools v0.0.0-20190514143549-2d081dbd584e // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/square/go-jose.v2 v2.3.1
 )

pkg/api/system.go

@@ -1,5 +1,7 @@
 package api
 
+import metrics "github.com/armon/go-metrics"
+
 type System struct {
 	client *Client
 }
@@ -37,3 +39,12 @@ func (s *System) Info() (*InfoResp, error) {
 	}
 	return &resp, nil
 }
+
+func (s *System) Metrics() (*metrics.MetricsSummary, error) {
+	var resp metrics.MetricsSummary
+	err := s.client.get("/v1/system/metrics", &resp)
+	if err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}

pkg/server/config.go

@@ -33,6 +33,8 @@ const (
 	routeDeleteJobGroupScalingPolicyPattern = "/v1/policy/{job_id}/{group}"
 	routeDeleteJobScalingPolicyName         = "DeleteJobScalingPolicy"
 	routeDeleteJobScalingPolicyPattern      = "/v1/policy/{job_id}"
+	routeGetMetricsName                     = "GetSystemMetrics"
+	routeGetMetricsPattern                  = "/v1/system/metrics"
 
 	telemetryInterval = 10
 )

pkg/server/routes.go

@@ -11,7 +11,7 @@ import (
 )
 
 type routes struct {
-	Health *systemV1.System
+	System *systemV1.System
 	Policy *policyV1.Policy
 	Scale  *scaleV1.Scale
 }
@@ -20,22 +20,28 @@ func (h *HTTPServer) setupRoutes() *router.RouteTable {
 	h.logger.Debug().Msg("setting up HTTP server routes")
 
 	// Setup our route servers with their required configuration.
-	h.routes.Health = systemV1.NewSystemServer(h.logger, h.nomad, h.cfg.Server)
+	h.routes.System = systemV1.NewSystemServer(h.logger, h.nomad, h.cfg.Server, h.telemetry)
 	h.routes.Scale = scaleV1.NewScaleServer(h.logger, h.cfg.Server.StrictPolicyChecking, h.policyBackend, h.nomad)
 	h.routes.Policy = policyV1.NewPolicyServer(h.logger, h.policyBackend)
 
-	healthRoutes := router.Routes{
+	systemRoutes := router.Routes{
 		router.Route{
 			Name:    routeSystemHealthName,
 			Method:  http.MethodGet,
 			Pattern: routeSystemHealthPattern,
-			Handler: h.routes.Health.GetHealth,
+			Handler: h.routes.System.GetHealth,
 		},
 		router.Route{
 			Name:    routeSystemInfoName,
 			Method:  http.MethodGet,
 			Pattern: routeSystemInfoPattern,
-			Handler: h.routes.Health.GetInfo,
+			Handler: h.routes.System.GetInfo,
+		},
+		router.Route{
+			Name:    routeGetMetricsName,
+			Method:  http.MethodGet,
+			Pattern: routeGetMetricsPattern,
+			Handler: h.routes.System.GetMetrics,
 		},
 	}
 
@@ -109,8 +115,8 @@ func (h *HTTPServer) setupRoutes() *router.RouteTable {
 				Handler: h.routes.Policy.DeleteJobPolicy,
 			},
 		}
-		return &router.RouteTable{healthRoutes, scaleRoutes, policyRoutes, apiPolicyEngineRoutes}
+		return &router.RouteTable{systemRoutes, scaleRoutes, policyRoutes, apiPolicyEngineRoutes}
 	}
 
-	return &router.RouteTable{healthRoutes, scaleRoutes, policyRoutes}
+	return &router.RouteTable{systemRoutes, scaleRoutes, policyRoutes}
 }

pkg/server/server.go

@@ -10,7 +10,7 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/armon/go-metrics"
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/nomad/api"
 	"github.com/jrasell/sherpa/pkg/autoscale"
 	"github.com/jrasell/sherpa/pkg/client"
@@ -70,6 +70,11 @@ func (h *HTTPServer) setup() error {
 		return err
 	}
 
+	// Setup telemetry based on the config passed by the operator.
+	if err := h.setupTelemetry(); err != nil {
+		return errors.Wrap(err, "failed to setup telemetry handler")
+	}
+
 	h.setupPolicyBackend()
 
 	// If the server has been set to enable the internal autoscaler, set this up and start the
@@ -83,11 +88,6 @@ func (h *HTTPServer) setup() error {
 	r := router.WithRoutes(h.logger, *initialRoutes)
 	http.Handle("/", middlewareLogger(r, h.logger))
 
-	// Setup telemetry based on the config passed by the operator.
-	if err := h.setupTelemetry(); err != nil {
-		return errors.Wrap(err, "failed to setup telemetry handler")
-	}
-
 	// Run the TLS setup process so that if the user has configured a TLS certificate pair the
 	// server uses these.
 	if err := h.setupTLS(); err != nil {

pkg/server/telemetry.go

@@ -3,7 +3,7 @@ package server
 import (
 	"time"
 
-	"github.com/armon/go-metrics"
+	metrics "github.com/armon/go-metrics"
 	"github.com/jrasell/sherpa/pkg/build"
 )
 

pkg/system/v1/system.go

@@ -1,13 +1,14 @@
 package v1
 
 import (
+	"encoding/json"
 	"net/http"
 
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/nomad/api"
 	serverCfg "github.com/jrasell/sherpa/pkg/config/server"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	"gopkg.in/square/go-jose.v2/json"
 )
 
 const (
@@ -23,9 +24,10 @@ const (
 )
 
 type System struct {
-	logger zerolog.Logger
-	nomad  *api.Client
-	server *serverCfg.Config
+	logger    zerolog.Logger
+	nomad     *api.Client
+	server    *serverCfg.Config
+	telemetry *metrics.InmemSink
 }
 
 type SystemInfoResp struct {
@@ -36,11 +38,12 @@ type SystemInfoResp struct {
 	StrictPolicyChecking      bool
 }
 
-func NewSystemServer(l zerolog.Logger, nomad *api.Client, server *serverCfg.Config) *System {
+func NewSystemServer(l zerolog.Logger, nomad *api.Client, server *serverCfg.Config, tel *metrics.InmemSink) *System {
 	return &System{
-		logger: l,
-		nomad:  nomad,
-		server: server,
+		logger:    l,
+		nomad:     nomad,
+		server:    server,
+		telemetry: tel,
 	}
 }
 
@@ -79,6 +82,24 @@ func (h *System) GetInfo(w http.ResponseWriter, r *http.Request) {
 	writeJSONResponse(w, out, http.StatusOK)
 }
 
+func (h *System) GetMetrics(w http.ResponseWriter, r *http.Request) {
+	metricData, err := h.telemetry.DisplayMetrics(w, r)
+	if err != nil {
+		h.logger.Error().Err(err).Msg("failed to get latest telemetry data")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	out, err := json.Marshal(metricData)
+	if err != nil {
+		h.logger.Error().Err(err).Msg("failed to marshal HTTP response")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	writeJSONResponse(w, out, http.StatusOK)
+}
+
 func writeJSONResponse(w http.ResponseWriter, bytes []byte, statusCode int) {
 	w.Header().Set(headerKeyContentType, headerValueContentTypeJSON)
 	w.WriteHeader(statusCode)

pkg/system/v1/system_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestSystem_GetHealth(t *testing.T) {
-	s := NewSystemServer(zerolog.Logger{}, nil, nil)
+	s := NewSystemServer(zerolog.Logger{}, nil, nil, nil)
 
 	r := httptest.NewRequest("GET", "http://jrasell.com/v1/system/health", nil)
 	w := httptest.NewRecorder()
@@ -55,7 +55,7 @@ func TestSystem_GetInfo(t *testing.T) {
 		r := httptest.NewRequest("GET", "http://jrasell.com/v1/system/info", nil)
 		w := httptest.NewRecorder()
 
-		s := NewSystemServer(zerolog.Logger{}, nomadClient, tc.systemServerConfig)
+		s := NewSystemServer(zerolog.Logger{}, nomadClient, tc.systemServerConfig, nil)
 		s.GetInfo(w, r)
 
 		assert.Equal(t, tc.expectedRespCode, w.Code)