Merge pull request #700 from ystia/feature/GH-688_ssh_connection_retries

Support ssh connection retries

CHANGELOG.md

@@ -17,6 +17,7 @@
 
 ### ENHANCEMENTS
 
+* Support ssh connection retries ([GH-688](https://github.com/ystia/yorc/issues/688))
 * Remove useless/cluttering logs ([GH-681](https://github.com/ystia/yorc/issues/681))
 * Should be able to specify edcsa or rsa ssh keys in gcloud compute instances metadata ([GH-697](https://github.com/ystia/yorc/issues/697))
 * Add the ability to define OpenStack Compute Instance metadata ([GH-687](https://github.com/ystia/yorc/issues/687))

commands/server.go

@@ -256,6 +256,8 @@ func setConfig() {
 	serverCmd.PersistentFlags().String("locations_file_path", "", "File path to locations configuration. This configuration is taken in account for the first time the server starts.")
 	serverCmd.PersistentFlags().Int("concurrency_limit_for_upgrades", config.DefaultUpgradesConcurrencyLimit, "Limit of concurrency used in Upgrade processes. If not set the default value will be used")
 	serverCmd.PersistentFlags().Duration("ssh_connection_timeout", config.DefaultSSHConnectionTimeout, "Timeout to establish SSH connection from Yorc SSH client. If not set the default value will be used")
+	serverCmd.PersistentFlags().Duration("ssh_connection_retry_backoff", config.DefaultSSHConnectionRetryBackoff, "Backoff duration before retrying an ssh connection. This may be superseded by a location attribute if supported.")
+	serverCmd.PersistentFlags().Uint64("ssh_connection_max_retries", config.DefaultSSHConnectionMaxRetries, "Maximum number of retries (attempts are retries + 1) before giving-up to connect. This may be superseded by a location attribute if supported.")
 
 	serverCmd.PersistentFlags().Duration("tasks_dispatcher_long_poll_wait_time", config.DefaultTasksDispatcherLongPollWaitTime, "Wait time when long polling for executions tasks to dispatch to workers")
 	serverCmd.PersistentFlags().Duration("tasks_dispatcher_lock_wait_time", config.DefaultTasksDispatcherLockWaitTime, "Wait time for acquiring a lock for an execution task")
@@ -321,6 +323,8 @@ func setConfig() {
 	viper.BindPFlag("locations_file_path", serverCmd.PersistentFlags().Lookup("locations_file_path"))
 	viper.BindPFlag("concurrency_limit_for_upgrades", serverCmd.PersistentFlags().Lookup("concurrency_limit_for_upgrades"))
 	viper.BindPFlag("ssh_connection_timeout", serverCmd.PersistentFlags().Lookup("ssh_connection_timeout"))
+	viper.BindPFlag("ssh_connection_retry_backoff", serverCmd.PersistentFlags().Lookup("ssh_connection_retry_backoff"))
+	viper.BindPFlag("ssh_connection_max_retries", serverCmd.PersistentFlags().Lookup("ssh_connection_max_retries"))
 
 	viper.BindPFlag("tasks.dispatcher.long_poll_wait_time", serverCmd.PersistentFlags().Lookup("tasks_dispatcher_long_poll_wait_time"))
 	viper.BindPFlag("tasks.dispatcher.lock_wait_time", serverCmd.PersistentFlags().Lookup("tasks_dispatcher_lock_wait_time"))
@@ -364,6 +368,8 @@ func setConfig() {
 	viper.BindEnv("locations_file_path")
 	viper.BindEnv("concurrency_limit_for_upgrades")
 	viper.BindEnv("ssh_connection_timeout")
+	viper.BindEnv("ssh_connection_retry_backoff")
+	viper.BindEnv("ssh_connection_max_retries")
 
 	//Bind Consul environment variables flags
 	for key := range consulConfiguration {
@@ -399,6 +405,8 @@ func setConfig() {
 	viper.SetDefault("disable_ssh_agent", false)
 	viper.SetDefault("concurrency_limit_for_upgrades", config.DefaultUpgradesConcurrencyLimit)
 	viper.SetDefault("ssh_connection_timeout", config.DefaultSSHConnectionTimeout)
+	viper.SetDefault("ssh_connection_retry_backoff", config.DefaultSSHConnectionRetryBackoff)
+	viper.SetDefault("ssh_connection_max_retries", config.DefaultSSHConnectionMaxRetries)
 
 	viper.SetDefault("tasks.dispatcher.long_poll_wait_time", config.DefaultTasksDispatcherLongPollWaitTime)
 	viper.SetDefault("tasks.dispatcher.lock_wait_time", config.DefaultTasksDispatcherLockWaitTime)

config/config.go

@@ -84,6 +84,12 @@ const DefaultUpgradesConcurrencyLimit = 1000
 // DefaultSSHConnectionTimeout is the default timeout for SSH connections
 const DefaultSSHConnectionTimeout = 10 * time.Second
 
+// DefaultSSHConnectionRetryBackoff is the default duration before retring connect for SSH connections
+const DefaultSSHConnectionRetryBackoff = 1 * time.Second
+
+// DefaultSSHConnectionMaxRetries is the default maximum number of retries before giving up (number of attempts is number of retries + 1)
+const DefaultSSHConnectionMaxRetries uint64 = 3
+
 // Configuration holds config information filled by Cobra and Viper (see commands package for more information)
 type Configuration struct {
 	Ansible                          Ansible       `yaml:"ansible,omitempty" mapstructure:"ansible"`
@@ -112,6 +118,8 @@ type Configuration struct {
 	Storage                          Storage       `yaml:"storage,omitempty" mapstructure:"storage"`
 	UpgradeConcurrencyLimit          int           `yaml:"concurrency_limit_for_upgrades,omitempty" mapstructure:"concurrency_limit_for_upgrades"`
 	SSHConnectionTimeout             time.Duration `yaml:"ssh_connection_timeout,omitempty" mapstructure:"ssh_connection_timeout"`
+	SSHConnectionRetryBackoff        time.Duration `yaml:"ssh_connection_retry_backoff,omitempty" mapstructure:"ssh_connection_retry_backoff"`
+	SSHConnectionMaxRetries          uint64        `yaml:"ssh_connection_max_retries,omitempty" mapstructure:"ssh_connection_max_retries"`
 }
 
 // DockerSandbox holds the configuration for a docker sandbox
@@ -298,18 +306,51 @@ func (dm DynamicMap) GetIntOrDefault(name string, defaultValue int) int {
 	return cast.ToInt(dm.Get(name))
 }
 
+// GetInt64OrDefault returns the value of the given key casted into an int64.
+// The given default value is returned if not found.
+func (dm DynamicMap) GetInt64OrDefault(name string, defaultValue int64) int64 {
+	if !dm.IsSet(name) {
+		return defaultValue
+	}
+	return cast.ToInt64(dm.Get(name))
+}
+
 // GetInt64 returns the value of the given key casted into an int64.
 // 0 is returned if not found.
 func (dm DynamicMap) GetInt64(name string) int64 {
 	return cast.ToInt64(dm.Get(name))
 }
 
+// GetUint64OrDefault returns the value of the given key casted into an uint64.
+// The given default value is returned if not found.
+func (dm DynamicMap) GetUint64OrDefault(name string, defaultValue uint64) uint64 {
+	if !dm.IsSet(name) {
+		return defaultValue
+	}
+	return cast.ToUint64(dm.Get(name))
+}
+
+// GetUint64 returns the value of the given key casted into an uint64.
+// 0 is returned if not found.
+func (dm DynamicMap) GetUint64(name string) uint64 {
+	return cast.ToUint64(dm.Get(name))
+}
+
 // GetDuration returns the value of the given key casted into a Duration.
 // A 0 duration is returned if not found.
 func (dm DynamicMap) GetDuration(name string) time.Duration {
 	return cast.ToDuration(dm.Get(name))
 }
 
+// GetDurationOrDefault returns the value of the given key casted into a Duration.
+// The given default value is returned if not found.
+func (dm DynamicMap) GetDurationOrDefault(name string, defaultValue time.Duration) time.Duration {
+	if !dm.IsSet(name) {
+		return defaultValue
+	}
+	return cast.ToDuration(dm.Get(name))
+}
+
 // DefaultConfigTemplateResolver is the default resolver for configuration templates
 var DefaultConfigTemplateResolver TemplateResolver = &configTemplateResolver{}