Skip to content

Commit

Permalink
BUG/MEDIUM: retry opening the runtime socket and allow delay start at…
Browse files Browse the repository at this point in the history
… startup
  • Loading branch information
hdurand0710 committed Apr 15, 2024
1 parent 9649a57 commit 853f111
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 38 deletions.
9 changes: 5 additions & 4 deletions client-native/cn.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
var runtimeClient runtime_api.Runtime

_, globalConf, err := confClient.GetGlobalConfiguration("")
waitForRuntimeOption := runtime_options.AllowDelayedStart(haproxyOptions.DelayedStartMax, haproxyOptions.DelayedStartTick)

// First try to setup master runtime socket
if err == nil {
Expand All @@ -87,15 +88,15 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
if globalConf.Nbproc > 0 {
nbproc := int(globalConf.Nbproc)
ms := runtime_options.MasterSocket(masterSocket, nbproc)
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms)
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption)
if err == nil {
return runtimeClient
}
log.Warningf("Error setting up runtime client with master socket: %s : %s", masterSocket, err.Error())
} else {
// if nbproc is not set, use master socket with 1 process
ms := runtime_options.MasterSocket(masterSocket, 1)
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms)
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption)
if err == nil {
return runtimeClient
}
Expand All @@ -110,7 +111,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
if misc.IsUnixSocketAddr(*r.Address) {
sockets[1] = *r.Address
socketsL := runtime_options.Sockets(sockets)
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL)
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL, waitForRuntimeOption)
if err == nil {
muSocketsList.Lock()
socketsList = sockets
Expand Down Expand Up @@ -143,7 +144,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
}

socketLst := runtime_options.Sockets(sockets)
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst)
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst, waitForRuntimeOption)
if err == nil {
return runtimeClient
}
Expand Down
71 changes: 37 additions & 34 deletions configuration/configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"path/filepath"
"strings"
"sync"
"time"

petname "github.com/dustinkirkland/golang-petname"
"github.com/haproxytech/client-native/v5/models"
Expand All @@ -34,40 +35,42 @@ import (
var cfg *Configuration

type HAProxyConfiguration struct {
SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"`
ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"`
HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"`
UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"`
ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"`
RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"`
StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"`
NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"`
PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"`
ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"`
TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"`
ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"`
BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"`
MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"`
SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"`
DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"`
ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"`
Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"`
MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"`
SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"`
GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"`
ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"`
UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"`
ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"`
MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"`
BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"`
ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"`
UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"`
GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"`
UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"`
ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"`
MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"`
DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"`
DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"`
SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"`
ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"`
HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"`
UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"`
ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"`
RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"`
StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"`
NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"`
PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"`
ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"`
TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"`
ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"`
BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"`
MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"`
SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"`
DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"`
ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"`
Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"`
MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"`
SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"`
GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"`
ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"`
UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"`
ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"`
MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"`
BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"`
ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"`
UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"`
GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"`
UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"`
ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"`
MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"`
DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"`
DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"`
DelayedStartMax time.Duration `long:"delayed-start-max" description:"Maximum duration to wait for the haproxy runtime socket to be ready" default:"30s" group:"haproxy"`
DelayedStartTick time.Duration `long:"delayed-start-tick" description:"Duration between checks for the haproxy runtime socket to be ready" default:"500ms" group:"haproxy"`
}

type User struct {
Expand Down
14 changes: 14 additions & 0 deletions configuration/configuration_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
package configuration

import (
"time"

"github.com/haproxytech/client-native/v5/models"
"github.com/jessevdk/go-flags"

Expand Down Expand Up @@ -74,6 +76,8 @@ type configTypeHaproxy struct {
NodeIDFile *string `yaml:"fid,omitempty"`
MasterWorkerMode *bool `yaml:"master_worker_mode,omitempty"`
Reload *configTypeReload `yaml:"reload,omitempty"`
DelayedStartMax *string `yaml:"delayed_start_max,omitempty"`
DelayedStartTick *string `yaml:"delayed_start_tick,omitempty"`
}

type configTypeUserlist struct {
Expand Down Expand Up @@ -391,6 +395,16 @@ func copyToConfiguration(cfg *Configuration) { //nolint:cyclop,maintidx
if cfgStorage.LogTargets != nil {
cfg.LogTargets = *cfgStorage.LogTargets
}
if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartMax != nil && !misc.HasOSArg("", "delayed-start-max", "") {
if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartMax); err == nil {
cfg.HAProxy.DelayedStartMax = d
}
}
if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartTick != nil && !misc.HasOSArg("", "delayed-start-tick", "") {
if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartTick); err == nil {
cfg.HAProxy.DelayedStartTick = d
}
}
}

func copyConfigurationToStorage(cfg *Configuration) {
Expand Down
2 changes: 2 additions & 0 deletions configuration/examples/example-full.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ haproxy:
master_runtime: null # string
fid: null # string
master_worker_mode: false # bool
delayed_start_max: 30s # time.Duration
delayed_start_tick: 500ms # time.Duration
reload:
reload_delay: 5 # int 2
reload_cmd: "systemctl reload haproxy"
Expand Down
2 changes: 2 additions & 0 deletions configure_data_plane.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import (
"github.com/haproxytech/dataplaneapi/operations/specification"
"github.com/haproxytech/dataplaneapi/operations/specification_openapiv3"
"github.com/haproxytech/dataplaneapi/rate"
"github.com/haproxytech/dataplaneapi/resilient"
socket_runtime "github.com/haproxytech/dataplaneapi/runtime"

// import various crypting algorithms
Expand Down Expand Up @@ -290,6 +291,7 @@ func configureAPI(api *operations.DataPlaneAPI) http.Handler { //nolint:cyclop,m
})

// setup transaction handlers
client = resilient.NewClient(client)
api.TransactionsStartTransactionHandler = &handlers.StartTransactionHandlerImpl{Client: client}
api.TransactionsDeleteTransactionHandler = &handlers.DeleteTransactionHandlerImpl{Client: client}
api.TransactionsGetTransactionHandler = &handlers.GetTransactionHandlerImpl{Client: client}
Expand Down
69 changes: 69 additions & 0 deletions resilient/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2019 HAProxy Technologies
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package resilient

import (
"context"
"errors"

client_native "github.com/haproxytech/client-native/v5"
"github.com/haproxytech/client-native/v5/runtime"
cn "github.com/haproxytech/dataplaneapi/client-native"
dataplaneapi_config "github.com/haproxytech/dataplaneapi/configuration"
)

type Client struct {
client_native.HAProxyClient
}

func NewClient(c client_native.HAProxyClient) *Client {
return &Client{
c,
}
}

// Runtime is a wrapper around HAProxyClient.Runtime
// that retries once to configure the runtime client if it failed
func (c *Client) Runtime() (runtime.Runtime, error) {
runtime, err := c.HAProxyClient.Runtime()

// We already have a valid runtime
// Let's return it
if err == nil {
return runtime, nil
}

// Now, for let's try to reconfigure once the runtime
cfg, err := c.HAProxyClient.Configuration()
if err != nil {
return nil, err
}

dpapiCfg := dataplaneapi_config.Get()
haproxyOptions := dpapiCfg.HAProxy

// Let's disable the delayed start by putting a max value to 0
// This is important to not block the handlers by waiting the DelayedStartMax that we wait for when we start
haproxyOptions.DelayedStartMax = 0
// let's retry
rnt := cn.ConfigureRuntimeClient(context.Background(), cfg, haproxyOptions)
if rnt == nil {
return nil, errors.New("retry - unable to configure runtime client")
}
c.HAProxyClient.ReplaceRuntime(rnt)

return c.HAProxyClient.Runtime()
}

0 comments on commit 853f111

Please sign in to comment.