Skip to content

Commit

Permalink
config: disable envoy admin by default, expose stats via envoy route (#…
Browse files Browse the repository at this point in the history 
…3677)
  • Loading branch information
@calebdoxsey
calebdoxsey committed Oct 18, 2022
1 parent 78d7a97 commit daed2d2
Show file tree
Hide file tree
Showing 11 changed files with 198 additions and 45 deletions.
18 changes: 15 additions & 3 deletions config/envoyconfig/bootstrap.go
View file
Expand Up @@ -3,10 +3,12 @@ package envoyconfig
import (
"fmt"
"os"
"path/filepath"

envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3"
envoy_extensions_access_loggers_file_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
Expand All @@ -17,15 +19,25 @@ import (
"github.com/pomerium/pomerium/internal/telemetry"
)

var (
envoyAdminAddressPath = filepath.Join(os.TempDir(), "pomerium-envoy-admin.sock")
envoyAdminAddressMode = 0o600
envoyAdminClusterName = "pomerium-envoy-admin"
)

// BuildBootstrapAdmin builds the admin config for the envoy bootstrap.
func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (admin *envoy_config_bootstrap_v3.Admin, err error) {
admin = &envoy_config_bootstrap_v3.Admin{
ProfilePath: cfg.Options.EnvoyAdminProfilePath,
}

admin.Address, err = parseAddress(cfg.Options.EnvoyAdminAddress)
if err != nil {
return nil, fmt.Errorf("envoyconfig: invalid envoy admin address: %w", err)
admin.Address = &envoy_config_core_v3.Address{
Address: &envoy_config_core_v3.Address_Pipe{
Pipe: &envoy_config_core_v3.Pipe{
Path: envoyAdminAddressPath,
Mode: uint32(envoyAdminAddressMode),
},
},
}

if cfg.Options.EnvoyAdminAccessLogPath != os.DevNull && cfg.Options.EnvoyAdminAccessLogPath != "" {
Expand Down
14 changes: 3 additions & 11 deletions config/envoyconfig/bootstrap_test.go
View file
Expand Up @@ -22,22 +22,14 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
testutil.AssertProtoJSONEqual(t, `
{
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9901
"pipe": {
"mode": 384,
"path": "`+envoyAdminAddressPath+`"
}
}
}
`, adminCfg)
})
t.Run("bad address", func(t *testing.T) {
_, err := b.BuildBootstrapAdmin(&config.Config{
Options: &config.Options{
EnvoyAdminAddress: "xyz1234:zyx4321",
},
})
assert.Error(t, err)
})
}

func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
Expand Down
6 changes: 6 additions & 0 deletions config/envoyconfig/clusters.go
View file
Expand Up @@ -79,13 +79,19 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
authorizeCluster.OutlierDetection = grpcAuthorizeOutlierDetection()
}

envoyAdminCluster, err := b.buildEnvoyAdminCluster(ctx, cfg)
if err != nil {
return nil, err
}

clusters := []*envoy_config_cluster_v3.Cluster{
b.buildACMETLSALPNCluster(cfg),
controlGRPC,
controlHTTP,
controlMetrics,
authorizeCluster,
databrokerCluster,
envoyAdminCluster,
}

tracingCluster, err := buildTracingCluster(cfg.Options)
Expand Down
36 changes: 36 additions & 0 deletions config/envoyconfig/clusters_envoy_admin.go
View file
@@ -0,0 +1,36 @@
package envoyconfig

import (
"context"

envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"

"github.com/pomerium/pomerium/config"
)

func (b *Builder) buildEnvoyAdminCluster(ctx context.Context, cfg *config.Config) (*envoy_config_cluster_v3.Cluster, error) {
return &envoy_config_cluster_v3.Cluster{
Name: envoyAdminClusterName,
ConnectTimeout: defaultConnectionTimeout,
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: envoyAdminClusterName,
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_config_core_v3.Address{
Address: &envoy_config_core_v3.Address_Pipe{
Pipe: &envoy_config_core_v3.Pipe{
Path: envoyAdminAddressPath,
},
},
},
},
},
}},
}},
},
}, nil
}
43 changes: 34 additions & 9 deletions config/envoyconfig/listeners.go
View file
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/pomerium/pomerium/internal/hashutil"
"github.com/pomerium/pomerium/internal/log"
"github.com/pomerium/pomerium/internal/sets"
"github.com/pomerium/pomerium/internal/telemetry/metrics"
"github.com/pomerium/pomerium/pkg/cryptutil"
)

Expand Down Expand Up @@ -80,6 +81,14 @@ func (b *Builder) BuildListeners(ctx context.Context, cfg *config.Config) ([]*en
listeners = append(listeners, li)
}

if cfg.Options.EnvoyAdminAddress != "" {
li, err := b.buildEnvoyAdminListener(ctx, cfg)
if err != nil {
return nil, err
}
listeners = append(listeners, li)
}

li, err := b.buildOutboundListener(cfg)
if err != nil {
return nil, err
Expand Down Expand Up @@ -381,19 +390,35 @@ func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_liste
rc, err := b.buildRouteConfiguration("metrics", []*envoy_config_route_v3.VirtualHost{{
Name: "metrics",
Domains: []string{"*"},
Routes: []*envoy_config_route_v3.Route{{
Name: "metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
Routes: []*envoy_config_route_v3.Route{
{
Name: "envoy-metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: metrics.EnvoyMetricsPath},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: envoyAdminClusterName,
},
PrefixRewrite: "/stats/prometheus",
},
},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-metrics",
{
Name: "metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-metrics",
},
},
},
},
}},
},
}})
if err != nil {
return nil, err
Expand Down
71 changes: 71 additions & 0 deletions config/envoyconfig/listeners_envoy_admin.go
View file
@@ -0,0 +1,71 @@
package envoyconfig

import (
"context"
"fmt"

envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"

"github.com/pomerium/pomerium/config"
)

func (b *Builder) buildEnvoyAdminListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter, err := b.buildEnvoyAdminHTTPConnectionManagerFilter()
if err != nil {
return nil, err
}

filterChain := &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{
filter,
},
}

addr, err := parseAddress(cfg.Options.EnvoyAdminAddress)
if err != nil {
return nil, fmt.Errorf("envoy_admin_addr %s: %w", cfg.Options.EnvoyAdminAddress, err)
}

li := newEnvoyListener("envoy-admin")
li.Address = addr
li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain}
return li, nil
}

func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
rc, err := b.buildRouteConfiguration("envoy-admin", []*envoy_config_route_v3.VirtualHost{{
Name: "envoy-admin",
Domains: []string{"*"},
Routes: []*envoy_config_route_v3.Route{
{
Name: "envoy-admin",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: envoyAdminClusterName,
},
},
},
},
},
}})
if err != nil {
return nil, err
}

return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
StatPrefix: "envoy-admin",
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
}), nil
}
26 changes: 19 additions & 7 deletions config/envoyconfig/listeners_test.go
View file
Expand Up @@ -63,15 +63,27 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
"virtualHosts": [{
"name": "metrics",
"domains": ["*"],
"routes": [{
"name": "metrics",
"match": {
"prefix": "/"
"routes": [
{
"name": "envoy-metrics",
"match": {
"prefix": "/metrics/envoy"
},
"route": {
"cluster": "pomerium-envoy-admin",
"prefixRewrite": "/stats/prometheus"
}
},
"route": {
"cluster": "pomerium-control-plane-metrics"
{
"name": "metrics",
"match": {
"prefix": "/"
},
"route": {
"cluster": "pomerium-control-plane-metrics"
}
}
}]
]
}]
},
"statPrefix": "metrics"
Expand Down
19 changes: 8 additions & 11 deletions config/metrics.go
View file
Expand Up @@ -24,14 +24,13 @@ const (

// A MetricsManager manages metrics for a given configuration.
type MetricsManager struct {
mu sync.RWMutex
installationID string
serviceName string
addr string
basicAuth string
envoyAdminAddress string
handler http.Handler
endpoints []MetricsScrapeEndpoint
mu sync.RWMutex
installationID string
serviceName string
addr string
basicAuth string
handler http.Handler
endpoints []MetricsScrapeEndpoint
}

// NewMetricsManager creates a new MetricsManager.
Expand Down Expand Up @@ -95,7 +94,6 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) {
mgr.addr = cfg.Options.MetricsAddr
mgr.basicAuth = cfg.Options.MetricsBasicAuth
mgr.installationID = cfg.Options.InstallationID
mgr.envoyAdminAddress = cfg.Options.EnvoyAdminAddress
mgr.handler = nil

if mgr.addr == "" {
Expand All @@ -106,7 +104,7 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) {
mgr.endpoints = append(cfg.MetricsScrapeEndpoints,
MetricsScrapeEndpoint{
Name: "envoy",
URL: url.URL{Scheme: "http", Host: cfg.Options.EnvoyAdminAddress, Path: "/stats/prometheus"},
URL: url.URL{Scheme: "http", Host: cfg.Options.MetricsAddr, Path: "/metrics/envoy"},
})
handler, err := metrics.PrometheusHandler(toInternalEndpoints(mgr.endpoints), mgr.installationID, defaultMetricsTimeout)
if err != nil {
Expand All @@ -125,7 +123,6 @@ func (mgr *MetricsManager) configUnchanged(cfg *Config) bool {
return cfg.Options.MetricsAddr == mgr.addr &&
cfg.Options.MetricsBasicAuth == mgr.basicAuth &&
cfg.Options.InstallationID == mgr.installationID &&
cfg.Options.EnvoyAdminAddress == mgr.envoyAdminAddress &&
reflect.DeepEqual(mgr.endpoints, cfg.MetricsScrapeEndpoints)
}

Expand Down
1 change: 0 additions & 1 deletion config/options.go
View file
Expand Up @@ -345,7 +345,6 @@ var defaultOptions = Options{
XffNumTrustedHops: 0,
EnvoyAdminAccessLogPath: os.DevNull,
EnvoyAdminProfilePath: os.DevNull,
EnvoyAdminAddress: "127.0.0.1:9901",
ProgrammaticRedirectDomainWhitelist: []string{"localhost"},
}

Expand Down
5 changes: 2 additions & 3 deletions config/options_test.go
View file
Expand Up @@ -316,7 +316,6 @@ func TestOptionsFromViper(t *testing.T) {
DataBrokerStorageType: "memory",
EnvoyAdminAccessLogPath: os.DevNull,
EnvoyAdminProfilePath: os.DevNull,
EnvoyAdminAddress: "127.0.0.1:9901",
},
false,
},
Expand All @@ -337,7 +336,6 @@ func TestOptionsFromViper(t *testing.T) {
DataBrokerStorageType: "memory",
EnvoyAdminAccessLogPath: os.DevNull,
EnvoyAdminProfilePath: os.DevNull,
EnvoyAdminAddress: "127.0.0.1:9901",
},
false,
},
Expand Down Expand Up @@ -414,7 +412,7 @@ func Test_AutoCertOptionsFromEnvVar(t *testing.T) {
cleanup func()
}

var tests = map[string]func(t *testing.T) test{
tests := map[string]func(t *testing.T) test{
"ok/simple": func(t *testing.T) test {
envs := map[string]string{
"AUTOCERT": "true",
Expand Down Expand Up @@ -689,6 +687,7 @@ func TestOptions_GetOauthOptions(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, u.Hostname(), oauthOptions.RedirectURL.Hostname())
}

func TestOptions_GetAllRouteableGRPCDomains(t *testing.T) {
opts := &Options{
AuthenticateURLString: "https://authenticate.example.com",
Expand Down

0 comments on commit daed2d2

Please sign in to comment.