Add config changes for UI metrics

.circleci/config.yml

@@ -626,7 +626,7 @@ jobs:
       - persist_to_workspace:
           root: .
           paths:
-            - ./agent/bindata_assetfs.go
+            - ./agent/uiserver/bindata_assetfs.go
       - run: *notify-slack-failure
 
   # commits static assets to git
@@ -641,16 +641,16 @@ jobs:
       - attach_workspace:
           at: .
       - run:
-          name: commit agent/bindata_assetfs.go if there are changes
+          name: commit agent/uiserver/bindata_assetfs.go if there are changes
           command: |
             exit 0
-            if ! git diff --exit-code agent/bindata_assetfs.go; then
+            if ! git diff --exit-code agent/uiserver/bindata_assetfs.go; then
               git config --local user.email "hashicorp-ci@users.noreply.github.com"
               git config --local user.name "hashicorp-ci"
 
               short_sha=$(git rev-parse --short HEAD)
-              git add agent/bindata_assetfs.go
-              git commit -m "auto-updated agent/bindata_assetfs.go from commit ${short_sha}"
+              git add agent/uiserver/bindata_assetfs.go
+              git commit -m "auto-updated agent/uiserver/bindata_assetfs.go from commit ${short_sha}"
               git push origin master
             else
               echo "no new static assets to publish"

GNUmakefile

@@ -16,7 +16,7 @@ GOARCH?=$(shell go env GOARCH)
 GOPATH=$(shell go env GOPATH)
 MAIN_GOPATH=$(shell go env GOPATH | cut -d: -f1)
 
-ASSETFS_PATH?=agent/bindata_assetfs.go
+ASSETFS_PATH?=agent/uiserver/bindata_assetfs.go
 # Get the git commit
 GIT_COMMIT?=$(shell git rev-parse --short HEAD)
 GIT_COMMIT_YEAR?=$(shell git show -s --format=%cd --date=format:%Y HEAD)
@@ -306,7 +306,7 @@ lint:
 # also run as part of the release build script when it verifies that there are no
 # changes to the UI assets that aren't checked in.
 static-assets:
-	@go-bindata-assetfs -modtime 1 -pkg agent -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/...
+	@go-bindata-assetfs -modtime 1 -pkg uiserver -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/...
 	@go fmt $(ASSETFS_PATH)
 
 

agent/agent.go

@@ -255,6 +255,23 @@ type Agent struct {
 	// fail, the agent will be shutdown.
 	apiServers *apiServers
 
+	// httpHandlers provides direct access to (one of) the HTTPHandlers started by
+	// this agent. This is used in tests to test HTTP endpoints without overhead
+	// of TCP connections etc.
+	//
+	// TODO: this is a temporary re-introduction after we removed a list of
+	// HTTPServers in favour of apiServers abstraction. Now that HTTPHandlers is
+	// stateful and has config reloading though it's not OK to just use a
+	// different instance of handlers in tests to the ones that the agent is wired
+	// up to since then config reloads won't actually affect the handlers under
+	// test while plumbing the external handlers in the TestAgent through bypasses
+	// testing that the agent itself is actually reloading the state correctly.
+	// Once we move `apiServers` to be a passed-in dependency for NewAgent, we
+	// should be able to remove this and have the Test Agent create the
+	// HTTPHandlers and pass them in removing the need to pull them back out
+	// again.
+	httpHandlers *HTTPHandlers
+
 	// wgServers is the wait group for all HTTP and DNS servers
 	// TODO: remove once dnsServers are handled by apiServers
 	wgServers sync.WaitGroup
@@ -290,6 +307,10 @@ type Agent struct {
 	// IP.
 	httpConnLimiter connlimit.Limiter
 
+	// configReloaders are subcomponents that need to be notified on a reload so
+	// they can update their internal state.
+	configReloaders []ConfigReloader
+
 	// enterpriseAgent embeds fields that we only access in consul-enterprise builds
 	enterpriseAgent
 }
@@ -735,6 +756,8 @@ func (a *Agent) listenHTTP() ([]apiServer, error) {
 				agent:    a,
 				denylist: NewDenylist(a.config.HTTPBlockEndpoints),
 			}
+			a.configReloaders = append(a.configReloaders, srv.ReloadConfig)
+			a.httpHandlers = srv
 			httpServer := &http.Server{
 				Addr:      l.Addr().String(),
 				TLSConfig: tlscfg,
@@ -3571,6 +3594,12 @@ func (a *Agent) reloadConfigInternal(newCfg *config.RuntimeConfig) error {
 
 	a.State.SetDiscardCheckOutput(newCfg.DiscardCheckOutput)
 
+	for _, r := range a.configReloaders {
+		if err := r(newCfg); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

agent/config/builder.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -797,6 +798,26 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		return RuntimeConfig{}, fmt.Errorf("serf_wan_allowed_cidrs: %s", err)
 	}
 
+	// Handle Deprecated UI config fields
+	if c.UI != nil {
+		b.warn("The 'ui' field is deprecated. Use the 'ui_config.enabled' field instead.")
+		if c.UIConfig.Enabled == nil {
+			c.UIConfig.Enabled = c.UI
+		}
+	}
+	if c.UIDir != nil {
+		b.warn("The 'ui_dir' field is deprecated. Use the 'ui_config.dir' field instead.")
+		if c.UIConfig.Dir == nil {
+			c.UIConfig.Dir = c.UIDir
+		}
+	}
+	if c.UIContentPath != nil {
+		b.warn("The 'ui_content_path' field is deprecated. Use the 'ui_config.content_path' field instead.")
+		if c.UIConfig.ContentPath == nil {
+			c.UIConfig.ContentPath = c.UIContentPath
+		}
+	}
+
 	// ----------------------------------------------------------------
 	// build runtime config
 	//
@@ -981,19 +1002,17 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		EnableDebug:                            b.boolVal(c.EnableDebug),
 		EnableRemoteScriptChecks:               enableRemoteScriptChecks,
 		EnableLocalScriptChecks:                enableLocalScriptChecks,
-
-		EnableUI:              b.boolVal(c.UI),
-		EncryptKey:            b.stringVal(c.EncryptKey),
-		EncryptVerifyIncoming: b.boolVal(c.EncryptVerifyIncoming),
-		EncryptVerifyOutgoing: b.boolVal(c.EncryptVerifyOutgoing),
-		GRPCPort:              grpcPort,
-		GRPCAddrs:             grpcAddrs,
-		HTTPMaxConnsPerClient: b.intVal(c.Limits.HTTPMaxConnsPerClient),
-		HTTPSHandshakeTimeout: b.durationVal("limits.https_handshake_timeout", c.Limits.HTTPSHandshakeTimeout),
-		KeyFile:               b.stringVal(c.KeyFile),
-		KVMaxValueSize:        b.uint64Val(c.Limits.KVMaxValueSize),
-		LeaveDrainTime:        b.durationVal("performance.leave_drain_time", c.Performance.LeaveDrainTime),
-		LeaveOnTerm:           leaveOnTerm,
+		EncryptKey:                             b.stringVal(c.EncryptKey),
+		EncryptVerifyIncoming:                  b.boolVal(c.EncryptVerifyIncoming),
+		EncryptVerifyOutgoing:                  b.boolVal(c.EncryptVerifyOutgoing),
+		GRPCPort:                               grpcPort,
+		GRPCAddrs:                              grpcAddrs,
+		HTTPMaxConnsPerClient:                  b.intVal(c.Limits.HTTPMaxConnsPerClient),
+		HTTPSHandshakeTimeout:                  b.durationVal("limits.https_handshake_timeout", c.Limits.HTTPSHandshakeTimeout),
+		KeyFile:                                b.stringVal(c.KeyFile),
+		KVMaxValueSize:                         b.uint64Val(c.Limits.KVMaxValueSize),
+		LeaveDrainTime:                         b.durationVal("performance.leave_drain_time", c.Performance.LeaveDrainTime),
+		LeaveOnTerm:                            leaveOnTerm,
 		Logging: logging.Config{
 			LogLevel:          b.stringVal(c.LogLevel),
 			LogJSON:           b.boolVal(c.LogJSON),
@@ -1058,8 +1077,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		TaggedAddresses:             c.TaggedAddresses,
 		TranslateWANAddrs:           b.boolVal(c.TranslateWANAddrs),
 		TxnMaxReqLen:                b.uint64Val(c.Limits.TxnMaxReqLen),
-		UIDir:                       b.stringVal(c.UIDir),
-		UIContentPath:               UIPathBuilder(b.stringVal(c.UIContentPath)),
+		UIConfig:                    b.uiConfigVal(c.UIConfig),
 		UnixSocketGroup:             b.stringVal(c.UnixSocket.Group),
 		UnixSocketMode:              b.stringVal(c.UnixSocket.Mode),
 		UnixSocketUser:              b.stringVal(c.UnixSocket.User),
@@ -1091,10 +1109,26 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 	return rt, nil
 }
 
+// reBasicName validates that a field contains only lower case alphanumerics,
+// underscore and dash and is non-empty.
+var reBasicName = regexp.MustCompile("^[a-z0-9_-]+$")
+
+func validateBasicName(field, value string, allowEmpty bool) error {
+	if value == "" {
+		if allowEmpty {
+			return nil
+		}
+		return fmt.Errorf("%s cannot be empty", field)
+	}
+	if !reBasicName.MatchString(value) {
+		return fmt.Errorf("%s can only contain lowercase alphanumeric, - or _ characters."+
+			" received: %q", field, value)
+	}
+	return nil
+}
+
 // Validate performs semantic validation of the runtime configuration.
 func (b *Builder) Validate(rt RuntimeConfig) error {
-	// reDatacenter defines a regexp for a valid datacenter name
-	var reDatacenter = regexp.MustCompile("^[a-z0-9_-]+$")
 
 	// validContentPath defines a regexp for a valid content path name.
 	var validContentPath = regexp.MustCompile(`^[A-Za-z0-9/_-]+$`)
@@ -1103,22 +1137,53 @@ func (b *Builder) Validate(rt RuntimeConfig) error {
 	// check required params we cannot recover from first
 	//
 
-	if rt.Datacenter == "" {
-		return fmt.Errorf("datacenter cannot be empty")
-	}
-	if !reDatacenter.MatchString(rt.Datacenter) {
-		return fmt.Errorf("datacenter cannot be %q. Please use only [a-z0-9-_]", rt.Datacenter)
+	if err := validateBasicName("datacenter", rt.Datacenter, false); err != nil {
+		return err
 	}
 	if rt.DataDir == "" && !rt.DevMode {
 		return fmt.Errorf("data_dir cannot be empty")
 	}
 
-	if !validContentPath.MatchString(rt.UIContentPath) {
-		return fmt.Errorf("ui-content-path can only contain alphanumeric, -, _, or /. received: %s", rt.UIContentPath)
+	if !validContentPath.MatchString(rt.UIConfig.ContentPath) {
+		return fmt.Errorf("ui-content-path can only contain alphanumeric, -, _, or /. received: %q", rt.UIConfig.ContentPath)
 	}
 
-	if hasVersion.MatchString(rt.UIContentPath) {
-		return fmt.Errorf("ui-content-path cannot have 'v[0-9]'. received: %s", rt.UIContentPath)
+	if hasVersion.MatchString(rt.UIConfig.ContentPath) {
+		return fmt.Errorf("ui-content-path cannot have 'v[0-9]'. received: %q", rt.UIConfig.ContentPath)
+	}
+
+	if err := validateBasicName("ui_config.metrics_provider", rt.UIConfig.MetricsProvider, true); err != nil {
+		return err
+	}
+	if rt.UIConfig.MetricsProviderOptionsJSON != "" {
+		// Attempt to parse the JSON to ensure it's valid, parsing into a map
+		// ensures we get an object.
+		var dummyMap map[string]interface{}
+		err := json.Unmarshal([]byte(rt.UIConfig.MetricsProviderOptionsJSON), &dummyMap)
+		if err != nil {
+			return fmt.Errorf("ui_config.metrics_provider_options_json must be empty "+
+				"or a string containing a valid JSON object. received: %q",
+				rt.UIConfig.MetricsProviderOptionsJSON)
+		}
+	}
+	if rt.UIConfig.MetricsProxy.BaseURL != "" {
+		u, err := url.Parse(rt.UIConfig.MetricsProxy.BaseURL)
+		if err != nil || !(u.Scheme == "http" || u.Scheme == "https") {
+			return fmt.Errorf("ui_config.metrics_proxy.base_url must be a valid http"+
+				" or https URL. received: %q",
+				rt.UIConfig.MetricsProxy.BaseURL)
+		}
+	}
+	for k, v := range rt.UIConfig.DashboardURLTemplates {
+		if err := validateBasicName("ui_config.dashboard_url_templates key names", k, false); err != nil {
+			return err
+		}
+		u, err := url.Parse(v)
+		if err != nil || !(u.Scheme == "http" || u.Scheme == "https") {
+			return fmt.Errorf("ui_config.dashboard_url_templates values must be a"+
+				" valid http or https URL. received: %q",
+				rt.UIConfig.MetricsProxy.BaseURL)
+		}
 	}
 
 	if !rt.DevMode {
@@ -1190,15 +1255,15 @@ func (b *Builder) Validate(rt RuntimeConfig) error {
 	if rt.AutopilotMaxTrailingLogs < 0 {
 		return fmt.Errorf("autopilot.max_trailing_logs cannot be %d. Must be greater than or equal to zero", rt.AutopilotMaxTrailingLogs)
 	}
-	if rt.ACLDatacenter != "" && !reDatacenter.MatchString(rt.ACLDatacenter) {
-		return fmt.Errorf("acl_datacenter cannot be %q. Please use only [a-z0-9-_]", rt.ACLDatacenter)
+	if err := validateBasicName("acl_datacenter", rt.ACLDatacenter, true); err != nil {
+		return err
 	}
 	// In DevMode, UI is enabled by default, so to enable rt.UIDir, don't perform this check
-	if !rt.DevMode && rt.EnableUI && rt.UIDir != "" {
+	if !rt.DevMode && rt.UIConfig.Enabled && rt.UIConfig.Dir != "" {
 		return fmt.Errorf(
-			"Both the ui and ui-dir flags were specified, please provide only one.\n" +
-				"If trying to use your own web UI resources, use the ui-dir flag.\n" +
-				"The web UI is included in the binary so use ui to enable it")
+			"Both the ui_config.enabled and ui_config.dir (or -ui and -ui-dir) were specified, please provide only one.\n" +
+				"If trying to use your own web UI resources, use ui_config.dir or the -ui-dir flag.\n" +
+				"The web UI is included in the binary so use ui_config.enabled or the -ui flag to enable it")
 	}
 	if rt.DNSUDPAnswerLimit < 0 {
 		return fmt.Errorf("dns_config.udp_answer_limit cannot be %d. Must be greater than or equal to zero", rt.DNSUDPAnswerLimit)
@@ -1647,6 +1712,35 @@ func (b *Builder) serviceConnectVal(v *ServiceConnect) *structs.ServiceConnect {
 	}
 }
 
+func (b *Builder) uiConfigVal(v RawUIConfig) UIConfig {
+	return UIConfig{
+		Enabled:                    b.boolVal(v.Enabled),
+		Dir:                        b.stringVal(v.Dir),
+		ContentPath:                UIPathBuilder(b.stringVal(v.ContentPath)),
+		MetricsProvider:            b.stringVal(v.MetricsProvider),
+		MetricsProviderFiles:       v.MetricsProviderFiles,
+		MetricsProviderOptionsJSON: b.stringVal(v.MetricsProviderOptionsJSON),
+		MetricsProxy:               b.uiMetricsProxyVal(v.MetricsProxy),
+		DashboardURLTemplates:      v.DashboardURLTemplates,
+	}
+}
+
+func (b *Builder) uiMetricsProxyVal(v RawUIMetricsProxy) UIMetricsProxy {
+	var hdrs []UIMetricsProxyAddHeader
+
+	for _, hdr := range v.AddHeaders {
+		hdrs = append(hdrs, UIMetricsProxyAddHeader{
+			Name:  b.stringVal(hdr.Name),
+			Value: b.stringVal(hdr.Value),
+		})
+	}
+
+	return UIMetricsProxy{
+		BaseURL:    b.stringVal(v.BaseURL),
+		AddHeaders: hdrs,
+	}
+}
+
 func (b *Builder) boolValWithDefault(v *bool, defaultVal bool) bool {
 	if v == nil {
 		return defaultVal