From 2a623a59d30f785b3a6d4834c6fe1d581bb2f628 Mon Sep 17 00:00:00 2001 From: TwiN Date: Wed, 7 Feb 2024 18:54:30 -0500 Subject: [PATCH] fix(web): Allow configuration of read-buffer-size (#675) This fixes the `431 Request Header Fields Too Large` error By default, the read-buffer-size is 8192, up from fiber's default of 4096. Fixes #674 Fixes #636 Supersedes #637 Supersedes #663 --- README.md | 110 +++++++++++++++++++++++------------------ api/api.go | 8 ++- config/web/web.go | 30 +++++++++-- config/web/web_test.go | 89 +++++++++++++++++++++++---------- 4 files changed, 157 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 6c028a54b..e58624df5 100644 --- a/README.md +++ b/README.md @@ -206,57 +206,58 @@ If you want to test it locally, see [Docker](#docker). ## Configuration | Parameter | Description | Default | |:------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------| -| `debug` | Whether to enable debug logs. | `false` | -| `metrics` | Whether to expose metrics at /metrics. | `false` | -| `storage` | [Storage configuration](#storage) | `{}` | -| `endpoints` | List of endpoints to monitor. | Required `[]` | -| `endpoints[].enabled` | Whether to monitor the endpoint. | `true` | -| `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` | -| `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard.
See [Endpoint groups](#endpoint-groups). | `""` | -| `endpoints[].url` | URL to send the request to. | Required `""` | -| `endpoints[].method` | Request method. | `GET` | -| `endpoints[].conditions` | Conditions used to determine the health of the endpoint.
See [Conditions](#conditions). | `[]` | -| `endpoints[].interval` | Duration to wait between every status check. | `60s` | -| `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` | -| `endpoints[].body` | Request body. | `""` | -| `endpoints[].headers` | Request headers. | `{}` | -| `endpoints[].dns` | Configuration for an endpoint of type DNS.
See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` | -| `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` | -| `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` | -| `endpoints[].ssh` | Configuration for an endpoint of type SSH.
See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` | +| `debug` | Whether to enable debug logs. | `false` | +| `metrics` | Whether to expose metrics at /metrics. | `false` | +| `storage` | [Storage configuration](#storage) | `{}` | +| `endpoints` | List of endpoints to monitor. | Required `[]` | +| `endpoints[].enabled` | Whether to monitor the endpoint. | `true` | +| `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` | +| `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard.
See [Endpoint groups](#endpoint-groups). | `""` | +| `endpoints[].url` | URL to send the request to. | Required `""` | +| `endpoints[].method` | Request method. | `GET` | +| `endpoints[].conditions` | Conditions used to determine the health of the endpoint.
See [Conditions](#conditions). | `[]` | +| `endpoints[].interval` | Duration to wait between every status check. | `60s` | +| `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` | +| `endpoints[].body` | Request body. | `""` | +| `endpoints[].headers` | Request headers. | `{}` | +| `endpoints[].dns` | Configuration for an endpoint of type DNS.
See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` | +| `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` | +| `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` | +| `endpoints[].ssh` | Configuration for an endpoint of type SSH.
See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` | | `endpoints[].ssh.username` | SSH username (e.g. example) | Required `""` | | `endpoints[].ssh.password` | SSH password (e.g. password) | Required `""` | -| `endpoints[].alerts[].type` | Type of alert.
See [Alerting](#alerting) for all valid types. | Required `""` | -| `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` | -| `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` | -| `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` | -| `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` | -| `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` | -| `endpoints[].client` | [Client configuration](#client-configuration). | `{}` | -| `endpoints[].ui` | UI configuration at the endpoint level. | `{}` | -| `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` | -| `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` | -| `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` | -| `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` | -| `alerting` | [Alerting configuration](#alerting). | `{}` | -| `security` | [Security configuration](#security). | `{}` | -| `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` | -| `skip-invalid-config-update` | Whether to ignore invalid configuration update.
See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` | -| `web` | Web configuration. | `{}` | -| `web.address` | Address to listen on. | `0.0.0.0` | -| `web.port` | Port to listen on. | `8080` | -| `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` | -| `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` | -| `ui` | UI configuration. | `{}` | -| `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` | -| `ui.description` | Meta description for the page. | `Gatus is an advanced...`. | -| `ui.header` | Header at the top of the dashboard. | `Health Status` | -| `ui.logo` | URL to the logo to display. | `""` | -| `ui.link` | Link to open when the logo is clicked. | `""` | -| `ui.buttons` | List of buttons to display below the header. | `[]` | -| `ui.buttons[].name` | Text to display on the button. | Required `""` | -| `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` | -| `maintenance` | [Maintenance configuration](#maintenance). | `{}` | +| `endpoints[].alerts[].type` | Type of alert.
See [Alerting](#alerting) for all valid types. | Required `""` | +| `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` | +| `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` | +| `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` | +| `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` | +| `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` | +| `endpoints[].client` | [Client configuration](#client-configuration). | `{}` | +| `endpoints[].ui` | UI configuration at the endpoint level. | `{}` | +| `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` | +| `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` | +| `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` | +| `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` | +| `alerting` | [Alerting configuration](#alerting). | `{}` | +| `security` | [Security configuration](#security). | `{}` | +| `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` | +| `skip-invalid-config-update` | Whether to ignore invalid configuration update.
See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` | +| `web` | Web configuration. | `{}` | +| `web.address` | Address to listen on. | `0.0.0.0` | +| `web.port` | Port to listen on. | `8080` | +| `web.read-buffer-size` | Buffer size for reading requests from a connection. Also limit for the maximum header size. | `8192` | +| `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` | +| `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` | +| `ui` | UI configuration. | `{}` | +| `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` | +| `ui.description` | Meta description for the page. | `Gatus is an advanced...`. | +| `ui.header` | Header at the top of the dashboard. | `Health Status` | +| `ui.logo` | URL to the logo to display. | `""` | +| `ui.link` | Link to open when the logo is clicked. | `""` | +| `ui.buttons` | List of buttons to display below the header. | `[]` | +| `ui.buttons[].name` | Text to display on the button. | Required `""` | +| `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` | +| `maintenance` | [Maintenance configuration](#maintenance). | `{}` | ### Conditions @@ -1909,6 +1910,17 @@ endpoints: +### How to fix 431 Request Header Fields Too Large error +Depending on where your environment is deployed and what kind of middleware or reverse proxy sits in front of Gatus, +you may run into this issue. This could be because the request headers are too large, e.g. big cookies. + +By default, `web.read-buffer-size` is set to `8192`, but increasing this value like so will increase the read buffer size: +```yaml +web: + read-buffer-size: 32768 +``` + + ### Badges #### Uptime ![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg) diff --git a/api/api.go b/api/api.go index b9e97a095..942679ba3 100644 --- a/api/api.go +++ b/api/api.go @@ -7,6 +7,7 @@ import ( "os" "github.com/TwiN/gatus/v5/config" + "github.com/TwiN/gatus/v5/config/web" static "github.com/TwiN/gatus/v5/web" "github.com/TwiN/health" fiber "github.com/gofiber/fiber/v2" @@ -26,6 +27,10 @@ type API struct { func New(cfg *config.Config) *API { api := &API{} + if cfg.Web == nil { + log.Println("[api.New] nil web config passed as parameter. This should only happen in tests. Using default web configuration") + cfg.Web = web.GetDefaultConfig() + } api.router = api.createRouter(cfg) return api } @@ -40,7 +45,8 @@ func (a *API) createRouter(cfg *config.Config) *fiber.App { log.Printf("[api.ErrorHandler] %s", err.Error()) return fiber.DefaultErrorHandler(c, err) }, - Network: fiber.NetworkTCP, + ReadBufferSize: cfg.Web.ReadBufferSize, + Network: fiber.NetworkTCP, }) if os.Getenv("ENVIRONMENT") == "dev" { app.Use(cors.New(cors.Config{ diff --git a/config/web/web.go b/config/web/web.go index fd3d20791..5110df021 100644 --- a/config/web/web.go +++ b/config/web/web.go @@ -13,10 +13,16 @@ const ( // DefaultPort is the default port the application will listen on DefaultPort = 8080 + + // DefaultReadBufferSize is the default value for ReadBufferSize + DefaultReadBufferSize = 8192 + + // MinimumReadBufferSize is the minimum value for ReadBufferSize, and also the default value set + // for fiber.Config.ReadBufferSize + MinimumReadBufferSize = 4096 ) -// Config is the structure which supports the configuration of the endpoint -// which provides access to the web frontend +// Config is the structure which supports the configuration of the server listening to requests type Config struct { // Address to listen on (defaults to 0.0.0.0 specified by DefaultAddress) Address string `yaml:"address"` @@ -24,6 +30,14 @@ type Config struct { // Port to listen on (default to 8080 specified by DefaultPort) Port int `yaml:"port"` + // ReadBufferSize sets fiber.Config.ReadBufferSize, which is the buffer size for reading requests coming from a + // single connection and also acts as a limit for the maximum header size. + // + // If you're getting occasional "Request Header Fields Too Large", you may want to try increasing this value. + // + // Defaults to DefaultReadBufferSize + ReadBufferSize int `yaml:"read-buffer-size,omitempty"` + // TLS configuration (optional) TLS *TLSConfig `yaml:"tls,omitempty"` } @@ -38,7 +52,11 @@ type TLSConfig struct { // GetDefaultConfig returns a Config struct with the default values func GetDefaultConfig() *Config { - return &Config{Address: DefaultAddress, Port: DefaultPort} + return &Config{ + Address: DefaultAddress, + Port: DefaultPort, + ReadBufferSize: DefaultReadBufferSize, + } } // ValidateAndSetDefaults validates the web configuration and sets the default values if necessary. @@ -53,6 +71,12 @@ func (web *Config) ValidateAndSetDefaults() error { } else if web.Port < 0 || web.Port > math.MaxUint16 { return fmt.Errorf("invalid port: value should be between %d and %d", 0, math.MaxUint16) } + // Validate ReadBufferSize + if web.ReadBufferSize == 0 { + web.ReadBufferSize = DefaultReadBufferSize // Not set? Use the default value. + } else if web.ReadBufferSize < MinimumReadBufferSize { + web.ReadBufferSize = MinimumReadBufferSize // Below the minimum? Use the minimum value. + } // Try to load the TLS certificates if web.TLS != nil { if err := web.TLS.isValid(); err != nil { diff --git a/config/web/web_test.go b/config/web/web_test.go index 858189264..63a34f968 100644 --- a/config/web/web_test.go +++ b/config/web/web_test.go @@ -12,6 +12,9 @@ func TestGetDefaultConfig(t *testing.T) { if defaultConfig.Address != DefaultAddress { t.Error("expected default config to have the default address") } + if defaultConfig.ReadBufferSize != DefaultReadBufferSize { + t.Error("expected default config to have the default read buffer size") + } if defaultConfig.TLS != nil { t.Error("expected default config to have TLS disabled") } @@ -19,18 +22,20 @@ func TestGetDefaultConfig(t *testing.T) { func TestConfig_ValidateAndSetDefaults(t *testing.T) { scenarios := []struct { - name string - cfg *Config - expectedAddress string - expectedPort int - expectedErr bool + name string + cfg *Config + expectedAddress string + expectedPort int + expectedReadBufferSize int + expectedErr bool }{ { - name: "no-explicit-config", - cfg: &Config{}, - expectedAddress: "0.0.0.0", - expectedPort: 8080, - expectedErr: false, + name: "no-explicit-config", + cfg: &Config{}, + expectedAddress: "0.0.0.0", + expectedPort: 8080, + expectedReadBufferSize: 8192, + expectedErr: false, }, { name: "invalid-port", @@ -38,25 +43,52 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) { expectedErr: true, }, { - name: "with-good-tls-config", - cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, - expectedAddress: "0.0.0.0", - expectedPort: 443, - expectedErr: false, + name: "read-buffer-size-below-minimum", + cfg: &Config{ReadBufferSize: 1024}, + expectedAddress: "0.0.0.0", + expectedPort: 8080, + expectedReadBufferSize: MinimumReadBufferSize, // minimum is 4096, default is 8192. + expectedErr: false, + }, + { + name: "read-buffer-size-at-minimum", + cfg: &Config{ReadBufferSize: MinimumReadBufferSize}, + expectedAddress: "0.0.0.0", + expectedPort: 8080, + expectedReadBufferSize: 4096, + expectedErr: false, + }, + { + name: "custom-read-buffer-size", + cfg: &Config{ReadBufferSize: 65536}, + expectedAddress: "0.0.0.0", + expectedPort: 8080, + expectedReadBufferSize: 65536, + expectedErr: false, + }, + { + name: "with-good-tls-config", + cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, + expectedAddress: "0.0.0.0", + expectedPort: 443, + expectedReadBufferSize: 8192, + expectedErr: false, }, { - name: "with-bad-tls-config", - cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, - expectedAddress: "0.0.0.0", - expectedPort: 443, - expectedErr: true, + name: "with-bad-tls-config", + cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, + expectedAddress: "0.0.0.0", + expectedPort: 443, + expectedReadBufferSize: 8192, + expectedErr: true, }, { - name: "with-partial-tls-config", - cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}}, - expectedAddress: "0.0.0.0", - expectedPort: 443, - expectedErr: true, + name: "with-partial-tls-config", + cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}}, + expectedAddress: "0.0.0.0", + expectedPort: 443, + expectedReadBufferSize: 8192, + expectedErr: true, }, } for _, scenario := range scenarios { @@ -68,10 +100,13 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) { } if !scenario.expectedErr { if scenario.cfg.Port != scenario.expectedPort { - t.Errorf("expected port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port) + t.Errorf("expected Port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port) + } + if scenario.cfg.ReadBufferSize != scenario.expectedReadBufferSize { + t.Errorf("expected ReadBufferSize to be %d, got %d", scenario.expectedReadBufferSize, scenario.cfg.ReadBufferSize) } if scenario.cfg.Address != scenario.expectedAddress { - t.Errorf("expected address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address) + t.Errorf("expected Address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address) } } })