Merge pull request #716 from ystia/feature/GH-715-dispatcher-metrics

Feature/gh 715 dispatcher metrics

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### FEATURES
 
+* Added a tasks dispatcher metrics refresh time configuration parameter ([GH-715](https://github.com/ystia/yorc/issues/715))
 * Added an ElasticSearch store for events and logs ([GH-658](https://github.com/ystia/yorc/issues/658))
 * [Slurm] Expose Slurm scontrol show job results as job attributes ([GH-664](https://github.com/ystia/yorc/issues/664))
 
@@ -31,6 +32,7 @@
 
 ### BUG FIXES
 
+* Yorc panics on ElasticSearch store error ([GH-719](https://github.com/ystia/yorc/issues/719))
 * Error when storing runtime attributes of google subnet ([GH-713](https://github.com/ystia/yorc/issues/713))
 * Bootstrap fails on hosts where a version of ansible < 2.10.0 is installed ([GH-695](https://github.com/ystia/yorc/issues/695))
 * Panic due to nil pointer dereference may happen when retrieving a workflow ([GH-691](https://github.com/ystia/yorc/issues/691))

commands/server.go

@@ -261,6 +261,7 @@ func setConfig() {
 
 	serverCmd.PersistentFlags().Duration("tasks_dispatcher_long_poll_wait_time", config.DefaultTasksDispatcherLongPollWaitTime, "Wait time when long polling for executions tasks to dispatch to workers")
 	serverCmd.PersistentFlags().Duration("tasks_dispatcher_lock_wait_time", config.DefaultTasksDispatcherLockWaitTime, "Wait time for acquiring a lock for an execution task")
+	serverCmd.PersistentFlags().Duration("tasks_dispatcher_metrics_refresh_time", config.DefaultTasksDispatcherMetricsRefreshTime, "Tasks dispatcher metrics refresh time")
 
 	// Flags definition for Yorc HTTP REST API
 	serverCmd.PersistentFlags().Int("http_port", config.DefaultHTTPPort, "Port number for the Yorc HTTP REST API. If omitted or set to '0' then the default port number is used, any positive integer will be used as it, and finally any negative value will let use a random port.")
@@ -328,6 +329,7 @@ func setConfig() {
 
 	viper.BindPFlag("tasks.dispatcher.long_poll_wait_time", serverCmd.PersistentFlags().Lookup("tasks_dispatcher_long_poll_wait_time"))
 	viper.BindPFlag("tasks.dispatcher.lock_wait_time", serverCmd.PersistentFlags().Lookup("tasks_dispatcher_lock_wait_time"))
+	viper.BindPFlag("tasks.dispatcher.metrics_refresh_time", serverCmd.PersistentFlags().Lookup("tasks_dispatcher_metrics_refresh_time"))
 
 	//Bind Flags Yorc HTTP REST API
 	viper.BindPFlag("http_port", serverCmd.PersistentFlags().Lookup("http_port"))
@@ -380,6 +382,7 @@ func setConfig() {
 	viper.BindEnv("purged_deployments_eviction_timeout")
 	viper.BindEnv("tasks.dispatcher.long_poll_wait_time")
 	viper.BindEnv("tasks.dispatcher.lock_wait_time")
+	viper.BindEnv("tasks.dispatcher.metrics_refresh_time")
 
 	//Bind Ansible environment variables flags
 	for key := range ansibleConfiguration {
@@ -410,6 +413,7 @@ func setConfig() {
 
 	viper.SetDefault("tasks.dispatcher.long_poll_wait_time", config.DefaultTasksDispatcherLongPollWaitTime)
 	viper.SetDefault("tasks.dispatcher.lock_wait_time", config.DefaultTasksDispatcherLockWaitTime)
+	viper.SetDefault("tasks.dispatcher.metrics_refresh_time", config.DefaultTasksDispatcherMetricsRefreshTime)
 
 	// Consul configuration default settings
 	for key, value := range consulConfiguration {

config/config.go

@@ -78,6 +78,9 @@ const DefaultTasksDispatcherLongPollWaitTime = 1 * time.Minute
 // DefaultTasksDispatcherLockWaitTime is the default wait time for acquiring a lock for an execution task
 const DefaultTasksDispatcherLockWaitTime = 50 * time.Millisecond
 
+// DefaultTasksDispatcherMetricsRefreshTime is the default refresh time for the Tasks dispatcher metrics
+const DefaultTasksDispatcherMetricsRefreshTime = 5 * time.Minute
+
 // DefaultUpgradesConcurrencyLimit is the default limit of concurrency used in Upgrade processes
 const DefaultUpgradesConcurrencyLimit = 1000
 
@@ -203,8 +206,9 @@ type Tasks struct {
 
 // Dispatcher configuration
 type Dispatcher struct {
-	LongPollWaitTime time.Duration `yaml:"long_poll_wait_time,omitempty" mapstructure:"long_poll_wait_time" json:"long_poll_wait_time,omitempty"`
-	LockWaitTime     time.Duration `yaml:"lock_wait_time,omitempty" mapstructure:"lock_wait_time" json:"lock_wait_time,omitempty"`
+	LongPollWaitTime   time.Duration `yaml:"long_poll_wait_time,omitempty" mapstructure:"long_poll_wait_time" json:"long_poll_wait_time,omitempty"`
+	LockWaitTime       time.Duration `yaml:"lock_wait_time,omitempty" mapstructure:"lock_wait_time" json:"lock_wait_time,omitempty"`
+	MetricsRefreshTime time.Duration `yaml:"metrics_refresh_time,omitempty" mapstructure:"metrics_refresh_time" json:"metrics_refresh_time,omitempty"`
 }
 
 // Storage configuration

doc/configuration.rst

@@ -190,6 +190,10 @@ Globals Command-line options
 
   * ``--tasks_dispatcher_lock_wait_time``: Wait time (Golang duration format) for acquiring a lock for an execution task. If not set the default value of `50ms` will be used.
 
+.. _option_tasks_dispatcher_metrics_refresh_time_cmd:
+
+  * ``--tasks_dispatcher_metrics_refresh_time``: Refresh time (Golang duration format) for the tasks dispatcher metrics. If not set the default value of `5m` will be used.
+
 .. _option_workers_cmd:
 
   * ``--workers_number``: Yorc instances use a pool of workers to handle deployment tasks. This option defines the size of this pool. If not set the default value of `30` will be used.
@@ -714,6 +718,7 @@ Below is an example of configuration file with Tasks configuration options.
       dispatcher:
         long_polling_wait_time: "1m"
         lock_wait_time: "50ms"
+        metrics_refresh_time: "5m"
 
 .. _option_tasks_dispatcher_long_polling_wait_time_cfg:
 
@@ -723,6 +728,11 @@ Below is an example of configuration file with Tasks configuration options.
 
   * ``lock_wait_time``: Equivalent to :ref:`--tasks_dispatcher_lock_wait_time <option_tasks_dispatcher_lock_wait_time_cmd>` command-line flag.
 
+.. _option_tasks_dispatcher_metrics_refresh_time_cfg:
+
+  * ``metrics_refresh_time``: Equivalent to :ref:`--tasks_dispatcher_metrics_refresh_time <option_tasks_dispatcher_metrics_refresh_time_cmd>` command-line flag.
+
+
 Environment variables
 ---------------------
 
@@ -862,6 +872,10 @@ Environment variables
 
   * ``YORC_TASKS_DISPATCHER_LOCK_WAIT_TIME``: Equivalent to :ref:`--tasks_dispatcher_lock_wait_time <option_tasks_dispatcher_lock_wait_time_cmd>` command-line flag.
 
+.. _option_tasks_dispatcher_metrics_refresh_time_env:
+
+  * ``YORC_TASKS_DISPATCHER_METRICS_REFRESH_TIME``: Equivalent to :ref:`--tasks_dispatcher_metrics_refresh_time <option_tasks_dispatcher_metrics_refresh_time_cmd>` command-line flag.
+
 .. _option_workers_env:
 
   * ``YORC_WORKERS_NUMBER``: Equivalent to :ref:`--workers_number <option_workers_cmd>` command-line flag.

pkg/Dockerfile

@@ -18,8 +18,8 @@ ADD rootfs /
 
 # Python is required here as it should not be removed automatically when uninstalling python-dev
 # We do not install the whole docker package but just docker-cli
-RUN apk add --update make openssh-client python3 python3-dev gcc musl-dev libffi-dev openssl-dev docker-cli && \
-    python3 -m ensurepip && \
+RUN apk add --update make openssh-client python3 python3-dev gcc musl-dev libffi-dev openssl-dev docker-cli cargo rust && \
+    python3 -m ensurepip --upgrade && \
     if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \
     if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
     pip install ansible==${ANSIBLE_VERSION} docker-py netaddr jmespath && \

tasks/workflow/dispatcher.go

@@ -82,7 +82,7 @@ func (d *Dispatcher) emitMetrics(client *api.Client) {
 		lastWarn := time.Now().Add(-6 * time.Minute)
 		for {
 			select {
-			case <-time.After(time.Second):
+			case <-time.After(d.cfg.Tasks.Dispatcher.MetricsRefreshTime):
 				d.emitWorkersMetrics()
 				d.emitTaskExecutionsMetrics(client, &lastWarn)
 			case <-d.shutdownCh:

tasks/workflow/dispatcher_test.go

@@ -87,8 +87,9 @@ func testDispatcherRun(t *testing.T, srv *testutil.TestServer, client *api.Clien
 		},
 		Tasks: config.Tasks{
 			Dispatcher: config.Dispatcher{
-				LongPollWaitTime: 50 * time.Millisecond,
-				LockWaitTime:     5 * time.Millisecond,
+				LongPollWaitTime:   50 * time.Millisecond,
+				LockWaitTime:       5 * time.Millisecond,
+				MetricsRefreshTime: 50 * time.Millisecond,
 			},
 		},
 	}