authenticate: fix debug and metrics endpoints (#3212)

config/config.go

@@ -18,6 +18,10 @@ type Config struct {
 	HTTPPort string
 	// OutboundPort is the port the outbound gRPC listener is running on.
 	OutboundPort string
+	// MetricsPort is the port the metrics listener is running on.
+	MetricsPort string
+	// DebugPort is the port the debug listener is running on.
+	DebugPort string
 }
 
 // Clone creates a clone of the config.
@@ -32,6 +36,8 @@ func (cfg *Config) Clone() *Config {
 		GRPCPort:     cfg.GRPCPort,
 		HTTPPort:     cfg.HTTPPort,
 		OutboundPort: cfg.OutboundPort,
+		MetricsPort:  cfg.MetricsPort,
+		DebugPort:    cfg.DebugPort,
 	}
 }
 

config/config_source.go

@@ -110,13 +110,15 @@ func NewFileOrEnvironmentSource(
 		return nil, err
 	}
 
-	ports, err := netutil.AllocatePorts(3)
+	ports, err := netutil.AllocatePorts(5)
 	if err != nil {
 		return nil, err
 	}
 	grpcPort := ports[0]
 	httpPort := ports[1]
 	outboundPort := ports[2]
+	metricsPort := ports[3]
+	debugPort := ports[4]
 
 	cfg := &Config{
 		Options:      options,
@@ -125,6 +127,8 @@ func NewFileOrEnvironmentSource(
 		GRPCPort:     grpcPort,
 		HTTPPort:     httpPort,
 		OutboundPort: outboundPort,
+		MetricsPort:  metricsPort,
+		DebugPort:    debugPort,
 	}
 	metrics.SetConfigInfo(ctx, cfg.Options.Services, "local", cfg.Checksum(), true)
 

config/envoyconfig/bootstrap_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	t.Run("valid", func(t *testing.T) {
 		adminCfg, err := b.BuildBootstrapAdmin(&config.Config{
 			Options: &config.Options{
@@ -41,7 +41,7 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
 }
 
 func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
-	b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil)
+	b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
 	staticCfg, err := b.BuildBootstrapLayeredRuntime()
 	assert.NoError(t, err)
 	testutil.AssertProtoJSONEqual(t, `
@@ -58,7 +58,7 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
 
 func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
 	t.Run("valid", func(t *testing.T) {
-		b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil)
+		b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
 		staticCfg, err := b.BuildBootstrapStaticResources()
 		assert.NoError(t, err)
 		testutil.AssertProtoJSONEqual(t, `
@@ -90,14 +90,14 @@ func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
 		`, staticCfg)
 	})
 	t.Run("bad gRPC address", func(t *testing.T) {
-		b := New("xyz:zyx", "localhost:2222", filemgr.NewManager(), nil)
+		b := New("xyz:zyx", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
 		_, err := b.BuildBootstrapStaticResources()
 		assert.Error(t, err)
 	})
 }
 
 func TestBuilder_BuildBootstrapStatsConfig(t *testing.T) {
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	t.Run("valid", func(t *testing.T) {
 		statsCfg, err := b.BuildBootstrapStatsConfig(&config.Config{
 			Options: &config.Options{

config/envoyconfig/builder.go

@@ -7,23 +7,26 @@ import (
 
 // A Builder builds envoy config from pomerium config.
 type Builder struct {
-	localGRPCAddress string
-	localHTTPAddress string
-	filemgr          *filemgr.Manager
-	reproxy          *reproxy.Handler
+	localGRPCAddress    string
+	localHTTPAddress    string
+	localMetricsAddress string
+	filemgr             *filemgr.Manager
+	reproxy             *reproxy.Handler
 }
 
 // New creates a new Builder.
 func New(
 	localGRPCAddress string,
 	localHTTPAddress string,
+	localMetricsAddress string,
 	fileManager *filemgr.Manager,
 	reproxyHandler *reproxy.Handler,
 ) *Builder {
 	return &Builder{
-		localGRPCAddress: localGRPCAddress,
-		localHTTPAddress: localHTTPAddress,
-		filemgr:          fileManager,
-		reproxy:          reproxyHandler,
+		localGRPCAddress:    localGRPCAddress,
+		localHTTPAddress:    localHTTPAddress,
+		localMetricsAddress: localMetricsAddress,
+		filemgr:             fileManager,
+		reproxy:             reproxyHandler,
 	}
 }

config/envoyconfig/clusters.go

@@ -34,6 +34,10 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		Scheme: "http",
 		Host:   b.localHTTPAddress,
 	}
+	metricsURL := &url.URL{
+		Scheme: "http",
+		Host:   b.localMetricsAddress,
+	}
 	authorizeURLs, err := cfg.Options.GetInternalAuthorizeURLs()
 	if err != nil {
 		return nil, err
@@ -53,6 +57,11 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		return nil, err
 	}
 
+	controlMetrics, err := b.buildInternalCluster(ctx, cfg.Options, "pomerium-control-plane-metrics", []*url.URL{metricsURL}, upstreamProtocolAuto)
+	if err != nil {
+		return nil, err
+	}
+
 	authorizeCluster, err := b.buildInternalCluster(ctx, cfg.Options, "pomerium-authorize", authorizeURLs, upstreamProtocolHTTP2)
 	if err != nil {
 		return nil, err
@@ -74,6 +83,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 	clusters := []*envoy_config_cluster_v3.Cluster{
 		controlGRPC,
 		controlHTTP,
+		controlMetrics,
 		authorizeCluster,
 		databrokerCluster,
 	}

config/envoyconfig/clusters_test.go

@@ -25,7 +25,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 	cacheDir, _ := os.UserCacheDir()
 	customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-32484c314b584447463735303142374c31414145374650305a525539554938594d524855353757313942494d473847535231.pem")
 
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	rootCABytes, _ := getCombinedCertificateAuthority("", "")
 	rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
 
@@ -379,7 +379,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 
 func Test_buildCluster(t *testing.T) {
 	ctx := context.Background()
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	rootCABytes, _ := getCombinedCertificateAuthority("", "")
 	rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
 	o1 := config.NewDefaultOptions()
@@ -858,7 +858,7 @@ func Test_bindConfig(t *testing.T) {
 	ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*10)
 	defer clearTimeout()
 
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	t.Run("no bind config", func(t *testing.T) {
 		cluster, err := b.buildPolicyCluster(ctx, &config.Options{}, &config.Policy{
 			From: "https://from.example.com",

config/envoyconfig/listeners.go

@@ -486,7 +486,7 @@ func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_liste
 			Action: &envoy_config_route_v3.Route_Route{
 				Route: &envoy_config_route_v3.RouteAction{
 					ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
-						Cluster: "pomerium-control-plane-http",
+						Cluster: "pomerium-control-plane-metrics",
 					},
 				},
 			},

config/envoyconfig/listeners_test.go

@@ -25,7 +25,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
 	certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
 	keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")
 
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	li, err := b.buildMetricsListener(&config.Config{
 		Options: &config.Options{
 			MetricsAddr:           "127.0.0.1:9902",
@@ -65,7 +65,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
 								"prefix": "/"
 							},
 							"route": {
-								"cluster": "pomerium-control-plane-http"
+								"cluster": "pomerium-control-plane-metrics"
 							}
 						}]
 					}]
@@ -108,7 +108,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
 }
 
 func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
-	b := New("local-grpc", "local-http", nil, nil)
+	b := New("local-grpc", "local-http", "local-metrics", nil, nil)
 
 	options := config.NewDefaultOptions()
 	options.SkipXffAppend = true
@@ -523,7 +523,7 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 }
 
 func Test_buildDownstreamTLSContext(t *testing.T) {
-	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 
 	cacheDir, _ := os.UserCacheDir()
 	certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
@@ -805,7 +805,7 @@ func Test_hostMatchesDomain(t *testing.T) {
 }
 
 func Test_buildRouteConfiguration(t *testing.T) {
-	b := New("local-grpc", "local-http", nil, nil)
+	b := New("local-grpc", "local-http", "local-metrics", nil, nil)
 	virtualHosts := make([]*envoy_config_route_v3.VirtualHost, 10)
 	routeConfig, err := b.buildRouteConfiguration("test-route-configuration", virtualHosts)
 	require.NoError(t, err)
@@ -815,7 +815,7 @@ func Test_buildRouteConfiguration(t *testing.T) {
 }
 
 func Test_requireProxyProtocol(t *testing.T) {
-	b := New("local-grpc", "local-http", nil, nil)
+	b := New("local-grpc", "local-http", "local-metrics", nil, nil)
 	t.Run("required", func(t *testing.T) {
 		li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
 			UseProxyProtocol: true,

config/envoyconfig/routes.go

@@ -57,51 +57,20 @@ func (b *Builder) buildPomeriumHTTPRoutes(options *config.Options, domain string
 		return nil, err
 	}
 	if !isFrontingAuthenticate {
-		// enable ext_authz
-		r, err := b.buildControlPlanePathRoute("/.pomerium/jwt", true)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-
-		// disable ext_authz and passthrough to proxy handlers
-		r, err = b.buildControlPlanePathRoute("/ping", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-		r, err = b.buildControlPlanePathRoute("/healthz", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-		r, err = b.buildControlPlanePathRoute("/.pomerium", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-		r, err = b.buildControlPlanePrefixRoute("/.pomerium/", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-		r, err = b.buildControlPlanePathRoute("/.well-known/pomerium", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
-		r, err = b.buildControlPlanePrefixRoute("/.well-known/pomerium/", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
+		routes = append(routes,
+			// enable ext_authz
+			b.buildControlPlanePathRoute("/.pomerium/jwt", true),
+			// disable ext_authz and passthrough to proxy handlers
+			b.buildControlPlanePathRoute("/ping", false),
+			b.buildControlPlanePathRoute("/healthz", false),
+			b.buildControlPlanePathRoute("/.pomerium", false),
+			b.buildControlPlanePrefixRoute("/.pomerium/", false),
+			b.buildControlPlanePathRoute("/.well-known/pomerium", false),
+			b.buildControlPlanePrefixRoute("/.well-known/pomerium/", false),
+		)
 		// per #837, only add robots.txt if there are no unauthenticated routes
 		if !hasPublicPolicyMatchingURL(options, url.URL{Scheme: "https", Host: domain, Path: "/robots.txt"}) {
-			r, err := b.buildControlPlanePathRoute("/robots.txt", false)
-			if err != nil {
-				return nil, err
-			}
-			routes = append(routes, r)
+			routes = append(routes, b.buildControlPlanePathRoute("/robots.txt", false))
 		}
 	}
 	// if we're handling authentication, add the oauth2 callback url
@@ -110,11 +79,10 @@ func (b *Builder) buildPomeriumHTTPRoutes(options *config.Options, domain string
 		return nil, err
 	}
 	if config.IsAuthenticate(options.Services) && hostMatchesDomain(authenticateURL, domain) {
-		r, err := b.buildControlPlanePrefixRoute("/", false)
-		if err != nil {
-			return nil, err
-		}
-		routes = append(routes, r)
+		routes = append(routes,
+			b.buildControlPlanePathRoute(options.AuthenticateCallbackPath, false),
+			b.buildControlPlanePathRoute("/", false),
+		)
 	}
 	// if we're the proxy and this is the forward-auth url
 	forwardAuthURL, err := options.GetForwardAuthURL()
@@ -196,7 +164,7 @@ func (b *Builder) buildControlPlanePathAndQueryRoute(path string, queryparams []
 	}, nil
 }
 
-func (b *Builder) buildControlPlanePathRoute(path string, protected bool) (*envoy_config_route_v3.Route, error) {
+func (b *Builder) buildControlPlanePathRoute(path string, protected bool) *envoy_config_route_v3.Route {
 	r := &envoy_config_route_v3.Route{
 		Name: "pomerium-path-" + path,
 		Match: &envoy_config_route_v3.RouteMatch{
@@ -215,10 +183,10 @@ func (b *Builder) buildControlPlanePathRoute(path string, protected bool) (*envo
 			"envoy.filters.http.ext_authz": disableExtAuthz,
 		}
 	}
-	return r, nil
+	return r
 }
 
-func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) (*envoy_config_route_v3.Route, error) {
+func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) *envoy_config_route_v3.Route {
 	r := &envoy_config_route_v3.Route{
 		Name: "pomerium-prefix-" + prefix,
 		Match: &envoy_config_route_v3.RouteMatch{
@@ -237,7 +205,7 @@ func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) (*
 			"envoy.filters.http.ext_authz": disableExtAuthz,
 		}
 	}
-	return r, nil
+	return r
 }
 
 // getClusterID returns a cluster ID

config/envoyconfig/routes_test.go

@@ -96,7 +96,8 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) {
 			`+routeString("path", "/.well-known/pomerium", false)+`,
 			`+routeString("prefix", "/.well-known/pomerium/", false)+`,
 			`+routeString("path", "/robots.txt", false)+`,
-			`+routeString("prefix", "/", false)+`
+			`+routeString("path", "/oauth2/callback", false)+`,
+			`+routeString("path", "/", false)+`
 		]`, routes)
 	})
 	t.Run("proxy fronting authenticate", func(t *testing.T) {
@@ -167,8 +168,7 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) {
 
 func Test_buildControlPlanePathRoute(t *testing.T) {
 	b := &Builder{filemgr: filemgr.NewManager()}
-	route, err := b.buildControlPlanePathRoute("/hello/world", false)
-	require.NoError(t, err)
+	route := b.buildControlPlanePathRoute("/hello/world", false)
 	testutil.AssertProtoJSONEqual(t, `
 		{
 			"name": "pomerium-path-/hello/world",
@@ -190,8 +190,7 @@ func Test_buildControlPlanePathRoute(t *testing.T) {
 
 func Test_buildControlPlanePrefixRoute(t *testing.T) {
 	b := &Builder{filemgr: filemgr.NewManager()}
-	route, err := b.buildControlPlanePrefixRoute("/hello/world/", false)
-	require.NoError(t, err)
+	route := b.buildControlPlanePrefixRoute("/hello/world/", false)
 	testutil.AssertProtoJSONEqual(t, `
 		{
 			"name": "pomerium-prefix-/hello/world/",