Skip to content

Commit

Permalink
enhancement: Advanced server configuration (#1218)
Browse files Browse the repository at this point in the history 
* enhancement: Advanced server configuration

Add options to configure server settings such as keepalive timeouts and
request size limits.

Signed-off-by: Charith Ellawala <charith@cerbos.dev>

* Fix idleTimeout yaml tag

Signed-off-by: Charith Ellawala <charith@cerbos.dev>

Signed-off-by: Charith Ellawala <charith@cerbos.dev>
  • Loading branch information
@charithe
charithe committed Sep 18, 2022
1 parent ef67d54 commit 0a84961
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 62 deletions.
10 changes: 10 additions & 0 deletions docs/modules/configuration/partials/fullconfiguration.adoc
View file
Expand Up @@ -42,6 +42,16 @@ server:
passwordHash: JDJ5JDEwJEdEOVFzZDE2VVhoVkR0N2VkUFBVM09nalc0QnNZaC9xc2E4bS9mcUJJcEZXenp5OUpjMi91Cgo= # PasswordHash is the base64-encoded bcrypt hash of the password to use for authentication.
username: cerbos # Username is the hardcoded username to use for authentication.
enabled: true # Enabled defines whether the admin API is enabled.
advanced: # Advanced server settings.
grpc: # GRPC server settings.
connectionTimeout: 60s # ConnectionTimeout sets the timeout for establishing a new connection.
maxConnectionAge: 600s # MaxConnectionAge sets the maximum age of a connection.
maxRecvMsgSizeBytes: 4194304 # MaxRecvMsgSizeBytes sets the maximum size of a single request message. Defaults to 4MiB. Affects performance and resource utilisation.
http: # HTTP server settings.
idleTimeout: 120s # IdleTimeout sets the keepalive timeout.
readHeaderTimeout: 15s # ReadHeaderTimeout sets the timeout for reading request headers.
readTimeout: 30s # ReadTimeout sets the timeout for reading a request.
writeTimeout: 30s # WriteTimeout sets the timeout for writing a response.
cors: # CORS defines the CORS configuration for the server.
allowedHeaders: ['content-type'] # AllowedHeaders is the contents of the allowed-headers header.
allowedOrigins: ['*'] # AllowedOrigins is the contents of the allowed-origins header.
Expand Down
70 changes: 60 additions & 10 deletions internal/server/conf.go
View file
Expand Up @@ -18,16 +18,23 @@ import (
)

const (
confKey = "server"
defaultHTTPListenAddr = ":3592"
defaultGRPCListenAddr = ":3593"
defaultAdminUsername = "cerbos"
defaultAdminPassword = "cerbosAdmin"
defaultRawAdminPasswordHash = "$2y$10$VlPwcwpgcGZ5KjTaN1Pzk.vpFiQVG6F2cSWzQa9RtrNo3IacbzsEi" //nolint:gosec
defaultMaxActionsPerResource = 50
defaultMaxResourcesPerRequest = 50
defaultUDSFileMode = "0o766"
requestItemsMax = 500
confKey = "server"
defaultAdminPassword = "cerbosAdmin"
defaultAdminUsername = "cerbos"
defaultGRPCConnectionTimeout = 60 * time.Second
defaultGRPCListenAddr = ":3593"
defaultGRPCMaxConnectionAge = 10 * time.Minute
defaultGRPCMaxRecvMsgSizeBytes = 4 * 1024 * 1024 // 4MiB
defaultHTTPIdleTimeout = 120 * time.Second
defaultHTTPListenAddr = ":3592"
defaultHTTPReadHeaderTimeout = 15 * time.Second
defaultHTTPReadTimeout = 30 * time.Second
defaultHTTPWriteTimeout = 30 * time.Second
defaultMaxActionsPerResource = 50
defaultMaxResourcesPerRequest = 50
defaultRawAdminPasswordHash = "$2y$10$VlPwcwpgcGZ5KjTaN1Pzk.vpFiQVG6F2cSWzQa9RtrNo3IacbzsEi" //nolint:gosec
defaultUDSFileMode = "0o766"
requestItemsMax = 500
)

var (
Expand Down Expand Up @@ -57,6 +64,8 @@ type Conf struct {
LogRequestPayloads bool `yaml:"logRequestPayloads" conf:",example=false"`
// PlaygroundEnabled defines whether the playground API is enabled.
PlaygroundEnabled bool `yaml:"playgroundEnabled" conf:",example=false"`
// Advanced server settings.
Advanced AdvancedConf `yaml:"advanced"`
}

// TLSConf holds TLS configuration.
Expand Down Expand Up @@ -125,6 +134,33 @@ type RequestLimitsConf struct {
MaxResourcesPerRequest uint `yaml:"maxResourcesPerRequest" conf:",example=50"`
}

type AdvancedConf struct {
// HTTP server settings.
HTTP AdvancedHTTPConf `yaml:"http"`
// GRPC server settings.
GRPC AdvancedGRPCConf `yaml:"grpc"`
}

type AdvancedHTTPConf struct {
// ReadTimeout sets the timeout for reading a request.
ReadTimeout time.Duration `yaml:"readTimeout" conf:",example=30s"`
// ReadHeaderTimeout sets the timeout for reading request headers.
ReadHeaderTimeout time.Duration `yaml:"readHeaderTimeout" conf:",example=15s"`
// WriteTimeout sets the timeout for writing a response.
WriteTimeout time.Duration `yaml:"writeTimeout" conf:",example=30s"`
// IdleTimeout sets the keepalive timeout.
IdleTimeout time.Duration `yaml:"idleTimeout" conf:",example=120s"`
}

type AdvancedGRPCConf struct {
// MaxRecvMsgSizeBytes sets the maximum size of a single request message. Defaults to 4MiB. Affects performance and resource utilisation.
MaxRecvMsgSizeBytes uint `yaml:"maxRecvMsgSizeBytes" conf:",example=4194304"`
// MaxConnectionAge sets the maximum age of a connection.
MaxConnectionAge time.Duration `yaml:"maxConnectionAge" conf:",example=600s"`
// ConnectionTimeout sets the timeout for establishing a new connection.
ConnectionTimeout time.Duration `yaml:"connectionTimeout" conf:",example=60s"`
}

func (c *Conf) Key() string {
return confKey
}
Expand All @@ -145,6 +181,20 @@ func (c *Conf) SetDefaults() {
PasswordHash: defaultAdminPasswordHash,
}
}

c.Advanced = AdvancedConf{
HTTP: AdvancedHTTPConf{
ReadTimeout: defaultHTTPReadTimeout,
ReadHeaderTimeout: defaultHTTPReadHeaderTimeout,
WriteTimeout: defaultHTTPWriteTimeout,
IdleTimeout: defaultHTTPIdleTimeout,
},
GRPC: AdvancedGRPCConf{
MaxRecvMsgSizeBytes: defaultGRPCMaxRecvMsgSizeBytes,
MaxConnectionAge: defaultGRPCMaxConnectionAge,
ConnectionTimeout: defaultGRPCConnectionTimeout,
},
}
}

func (c *Conf) Validate() (errs error) {
Expand Down
12 changes: 7 additions & 5 deletions internal/server/server.go
View file
Expand Up @@ -89,7 +89,6 @@ import (

const (
defaultTimeout = 30 * time.Second
maxConnectionAge = 10 * time.Minute
metricsReportingInterval = 15 * time.Second
minGRPCConnectTimeout = 20 * time.Second

Expand Down Expand Up @@ -440,7 +439,9 @@ func (s *Server) mkGRPCServer(log *zap.Logger, auditLog audit.Log) (*grpc.Server
auditInterceptor,
),
grpc.StatsHandler(&ocgrpc.ServerHandler{}),
grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnectionAge}),
grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: s.conf.Advanced.GRPC.MaxConnectionAge}),
grpc.ConnectionTimeout(s.conf.Advanced.GRPC.ConnectionTimeout),
grpc.MaxRecvMsgSize(int(s.conf.Advanced.GRPC.MaxRecvMsgSizeBytes)),
grpc.UnknownServiceHandler(handleUnknownServices),
}

Expand Down Expand Up @@ -515,9 +516,10 @@ func (s *Server) startHTTPServer(ctx context.Context, l net.Listener, grpcSrv *g
h := &http.Server{
ErrorLog: zap.NewStdLog(zap.L().Named("http.error")),
Handler: h2c.NewHandler(httpHandler, &http2.Server{}),
ReadHeaderTimeout: defaultTimeout,
ReadTimeout: defaultTimeout,
WriteTimeout: defaultTimeout,
ReadHeaderTimeout: s.conf.Advanced.HTTP.ReadHeaderTimeout,
ReadTimeout: s.conf.Advanced.HTTP.ReadTimeout,
WriteTimeout: s.conf.Advanced.HTTP.WriteTimeout,
IdleTimeout: s.conf.Advanced.HTTP.IdleTimeout,
}

s.group.Go(func() error {
Expand Down
95 changes: 48 additions & 47 deletions internal/server/server_test.go
View file
Expand Up @@ -67,16 +67,15 @@ func TestServer(t *testing.T) {
testdataDir := test.PathToDir(t, "server")

t.Run("tcp", func(t *testing.T) {
conf := &Conf{
HTTPListenAddr: getFreeListenAddr(t),
GRPCListenAddr: getFreeListenAddr(t),
TLS: &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
},
PlaygroundEnabled: true,
RequestLimits: reqLimits,
conf := defaultConf()
conf.HTTPListenAddr = getFreeListenAddr(t)
conf.GRPCListenAddr = getFreeListenAddr(t)
conf.TLS = &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
}
conf.PlaygroundEnabled = true
conf.RequestLimits = reqLimits

ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
Expand All @@ -93,16 +92,15 @@ func TestServer(t *testing.T) {
t.Run("uds", func(t *testing.T) {
tempDir := t.TempDir()

conf := &Conf{
HTTPListenAddr: fmt.Sprintf("unix:%s", filepath.Join(tempDir, "http.sock")),
GRPCListenAddr: fmt.Sprintf("unix:%s", filepath.Join(tempDir, "grpc.sock")),
TLS: &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
},
PlaygroundEnabled: true,
RequestLimits: reqLimits,
conf := defaultConf()
conf.HTTPListenAddr = fmt.Sprintf("unix:%s", filepath.Join(tempDir, "http.sock"))
conf.GRPCListenAddr = fmt.Sprintf("unix:%s", filepath.Join(tempDir, "grpc.sock"))
conf.TLS = &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
}
conf.PlaygroundEnabled = true
conf.RequestLimits = reqLimits

ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
Expand All @@ -118,12 +116,11 @@ func TestServer(t *testing.T) {

t.Run("without_tls", func(t *testing.T) {
t.Run("tcp", func(t *testing.T) {
conf := &Conf{
HTTPListenAddr: getFreeListenAddr(t),
GRPCListenAddr: getFreeListenAddr(t),
PlaygroundEnabled: true,
RequestLimits: reqLimits,
}
conf := defaultConf()
conf.HTTPListenAddr = getFreeListenAddr(t)
conf.GRPCListenAddr = getFreeListenAddr(t)
conf.PlaygroundEnabled = true
conf.RequestLimits = reqLimits

ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
Expand All @@ -138,12 +135,11 @@ func TestServer(t *testing.T) {
t.Run("uds", func(t *testing.T) {
tempDir := t.TempDir()

conf := &Conf{
HTTPListenAddr: fmt.Sprintf("unix:%s", filepath.Join(tempDir, "http.sock")),
GRPCListenAddr: fmt.Sprintf("unix:%s", filepath.Join(tempDir, "grpc.sock")),
PlaygroundEnabled: true,
RequestLimits: reqLimits,
}
conf := defaultConf()
conf.HTTPListenAddr = fmt.Sprintf("unix:%s", filepath.Join(tempDir, "http.sock"))
conf.GRPCListenAddr = fmt.Sprintf("unix:%s", filepath.Join(tempDir, "grpc.sock"))
conf.PlaygroundEnabled = true
conf.RequestLimits = reqLimits

ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
Expand Down Expand Up @@ -182,25 +178,24 @@ func TestAdminService(t *testing.T) {
require.NoError(t, err)

testdataDir := test.PathToDir(t, "server")
conf := &Conf{
HTTPListenAddr: getFreeListenAddr(t),
GRPCListenAddr: getFreeListenAddr(t),
TLS: &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
},
AdminAPI: AdminAPIConf{
Enabled: true,
AdminCredentials: &AdminCredentialsConf{
Username: "cerbos",
PasswordHash: base64.StdEncoding.EncodeToString([]byte("$2y$10$yOdMOoQq6g7s.ogYRBDG3e2JyJFCyncpOEmkEyV.mNGKNyg68uPZS")),
},
},
RequestLimits: RequestLimitsConf{
MaxActionsPerResource: 5,
MaxResourcesPerRequest: 5,
conf := defaultConf()
conf.HTTPListenAddr = getFreeListenAddr(t)
conf.GRPCListenAddr = getFreeListenAddr(t)
conf.TLS = &TLSConf{
Cert: filepath.Join(testdataDir, "tls.crt"),
Key: filepath.Join(testdataDir, "tls.key"),
}
conf.AdminAPI = AdminAPIConf{
Enabled: true,
AdminCredentials: &AdminCredentialsConf{
Username: "cerbos",
PasswordHash: base64.StdEncoding.EncodeToString([]byte("$2y$10$yOdMOoQq6g7s.ogYRBDG3e2JyJFCyncpOEmkEyV.mNGKNyg68uPZS")),
},
}
conf.RequestLimits = RequestLimitsConf{
MaxActionsPerResource: 5,
MaxResourcesPerRequest: 5,
}

startServer(ctx, conf, Param{Store: store, Engine: eng, AuditLog: auditLog, AuxData: auxData})

Expand Down Expand Up @@ -233,3 +228,9 @@ func startServer(ctx context.Context, conf *Conf, param Param) {
}()
runtime.Gosched()
}

func defaultConf() *Conf {
conf := &Conf{}
conf.SetDefaults()
return conf
}

0 comments on commit 0a84961

Please sign in to comment.