From 853f111457fa02baf114b19cba73ce97ace9681e Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Thu, 11 Apr 2024 17:43:29 +0200 Subject: [PATCH] BUG/MEDIUM: retry opening the runtime socket and allow delay start at startup --- client-native/cn.go | 9 +-- configuration/configuration.go | 71 ++++++++++++------------ configuration/configuration_storage.go | 14 +++++ configuration/examples/example-full.yaml | 2 + configure_data_plane.go | 2 + resilient/client.go | 69 +++++++++++++++++++++++ 6 files changed, 129 insertions(+), 38 deletions(-) create mode 100644 resilient/client.go diff --git a/client-native/cn.go b/client-native/cn.go index 2afb9251..86df2dd6 100644 --- a/client-native/cn.go +++ b/client-native/cn.go @@ -76,6 +76,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config var runtimeClient runtime_api.Runtime _, globalConf, err := confClient.GetGlobalConfiguration("") + waitForRuntimeOption := runtime_options.AllowDelayedStart(haproxyOptions.DelayedStartMax, haproxyOptions.DelayedStartTick) // First try to setup master runtime socket if err == nil { @@ -87,7 +88,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config if globalConf.Nbproc > 0 { nbproc := int(globalConf.Nbproc) ms := runtime_options.MasterSocket(masterSocket, nbproc) - runtimeClient, err = runtime_api.New(ctx, mapsDir, ms) + runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption) if err == nil { return runtimeClient } @@ -95,7 +96,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config } else { // if nbproc is not set, use master socket with 1 process ms := runtime_options.MasterSocket(masterSocket, 1) - runtimeClient, err = runtime_api.New(ctx, mapsDir, ms) + runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption) if err == nil { return runtimeClient } @@ -110,7 +111,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config if misc.IsUnixSocketAddr(*r.Address) { sockets[1] = *r.Address socketsL := runtime_options.Sockets(sockets) - runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL) + runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL, waitForRuntimeOption) if err == nil { muSocketsList.Lock() socketsList = sockets @@ -143,7 +144,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config } socketLst := runtime_options.Sockets(sockets) - runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst) + runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst, waitForRuntimeOption) if err == nil { return runtimeClient } diff --git a/configuration/configuration.go b/configuration/configuration.go index 5ab12156..6bb4a202 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -24,6 +24,7 @@ import ( "path/filepath" "strings" "sync" + "time" petname "github.com/dustinkirkland/golang-petname" "github.com/haproxytech/client-native/v5/models" @@ -34,40 +35,42 @@ import ( var cfg *Configuration type HAProxyConfiguration struct { - SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"` - ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"` - HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"` - UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"` - ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"` - RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"` - StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"` - NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"` - PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"` - ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"` - TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"` - ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"` - BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"` - MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"` - SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"` - DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"` - ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"` - Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"` - MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"` - SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"` - GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"` - ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"` - UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"` - ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"` - MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"` - BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"` - ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"` - UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"` - GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"` - UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"` - ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"` - MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"` - DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"` - DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"` + SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"` + ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"` + HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"` + UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"` + ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"` + RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"` + StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"` + NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"` + PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"` + ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"` + TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"` + ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"` + BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"` + MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"` + SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"` + DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"` + ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"` + Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"` + MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"` + SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"` + GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"` + ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"` + UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"` + ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"` + MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"` + BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"` + ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"` + UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"` + GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"` + UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"` + ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"` + MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"` + DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"` + DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"` + DelayedStartMax time.Duration `long:"delayed-start-max" description:"Maximum duration to wait for the haproxy runtime socket to be ready" default:"30s" group:"haproxy"` + DelayedStartTick time.Duration `long:"delayed-start-tick" description:"Duration between checks for the haproxy runtime socket to be ready" default:"500ms" group:"haproxy"` } type User struct { diff --git a/configuration/configuration_storage.go b/configuration/configuration_storage.go index 588e1a71..3b1c3761 100644 --- a/configuration/configuration_storage.go +++ b/configuration/configuration_storage.go @@ -16,6 +16,8 @@ package configuration import ( + "time" + "github.com/haproxytech/client-native/v5/models" "github.com/jessevdk/go-flags" @@ -74,6 +76,8 @@ type configTypeHaproxy struct { NodeIDFile *string `yaml:"fid,omitempty"` MasterWorkerMode *bool `yaml:"master_worker_mode,omitempty"` Reload *configTypeReload `yaml:"reload,omitempty"` + DelayedStartMax *string `yaml:"delayed_start_max,omitempty"` + DelayedStartTick *string `yaml:"delayed_start_tick,omitempty"` } type configTypeUserlist struct { @@ -391,6 +395,16 @@ func copyToConfiguration(cfg *Configuration) { //nolint:cyclop,maintidx if cfgStorage.LogTargets != nil { cfg.LogTargets = *cfgStorage.LogTargets } + if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartMax != nil && !misc.HasOSArg("", "delayed-start-max", "") { + if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartMax); err == nil { + cfg.HAProxy.DelayedStartMax = d + } + } + if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartTick != nil && !misc.HasOSArg("", "delayed-start-tick", "") { + if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartTick); err == nil { + cfg.HAProxy.DelayedStartTick = d + } + } } func copyConfigurationToStorage(cfg *Configuration) { diff --git a/configuration/examples/example-full.yaml b/configuration/examples/example-full.yaml index 92acf832..d5674b2d 100644 --- a/configuration/examples/example-full.yaml +++ b/configuration/examples/example-full.yaml @@ -59,6 +59,8 @@ haproxy: master_runtime: null # string fid: null # string master_worker_mode: false # bool + delayed_start_max: 30s # time.Duration + delayed_start_tick: 500ms # time.Duration reload: reload_delay: 5 # int 2 reload_cmd: "systemctl reload haproxy" diff --git a/configure_data_plane.go b/configure_data_plane.go index 53d44cae..1c31a88d 100644 --- a/configure_data_plane.go +++ b/configure_data_plane.go @@ -58,6 +58,7 @@ import ( "github.com/haproxytech/dataplaneapi/operations/specification" "github.com/haproxytech/dataplaneapi/operations/specification_openapiv3" "github.com/haproxytech/dataplaneapi/rate" + "github.com/haproxytech/dataplaneapi/resilient" socket_runtime "github.com/haproxytech/dataplaneapi/runtime" // import various crypting algorithms @@ -290,6 +291,7 @@ func configureAPI(api *operations.DataPlaneAPI) http.Handler { //nolint:cyclop,m }) // setup transaction handlers + client = resilient.NewClient(client) api.TransactionsStartTransactionHandler = &handlers.StartTransactionHandlerImpl{Client: client} api.TransactionsDeleteTransactionHandler = &handlers.DeleteTransactionHandlerImpl{Client: client} api.TransactionsGetTransactionHandler = &handlers.GetTransactionHandlerImpl{Client: client} diff --git a/resilient/client.go b/resilient/client.go new file mode 100644 index 00000000..2e512944 --- /dev/null +++ b/resilient/client.go @@ -0,0 +1,69 @@ +// Copyright 2019 HAProxy Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package resilient + +import ( + "context" + "errors" + + client_native "github.com/haproxytech/client-native/v5" + "github.com/haproxytech/client-native/v5/runtime" + cn "github.com/haproxytech/dataplaneapi/client-native" + dataplaneapi_config "github.com/haproxytech/dataplaneapi/configuration" +) + +type Client struct { + client_native.HAProxyClient +} + +func NewClient(c client_native.HAProxyClient) *Client { + return &Client{ + c, + } +} + +// Runtime is a wrapper around HAProxyClient.Runtime +// that retries once to configure the runtime client if it failed +func (c *Client) Runtime() (runtime.Runtime, error) { + runtime, err := c.HAProxyClient.Runtime() + + // We already have a valid runtime + // Let's return it + if err == nil { + return runtime, nil + } + + // Now, for let's try to reconfigure once the runtime + cfg, err := c.HAProxyClient.Configuration() + if err != nil { + return nil, err + } + + dpapiCfg := dataplaneapi_config.Get() + haproxyOptions := dpapiCfg.HAProxy + + // Let's disable the delayed start by putting a max value to 0 + // This is important to not block the handlers by waiting the DelayedStartMax that we wait for when we start + haproxyOptions.DelayedStartMax = 0 + // let's retry + rnt := cn.ConfigureRuntimeClient(context.Background(), cfg, haproxyOptions) + if rnt == nil { + return nil, errors.New("retry - unable to configure runtime client") + } + c.HAProxyClient.ReplaceRuntime(rnt) + + return c.HAProxyClient.Runtime() +}