fix #279 - properly utilize the service name to restart tasks

stelligent · Jul 30, 2018 · 9fd5844 · 9fd5844
1 parent f7d0d65
commit 9fd5844
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/common/types.go b/common/types.go
@@ -320,6 +320,7 @@ type Task struct {
 	Name           string
 	Environment    string
 	Service        string
+	Status         string
 	TaskDefinition string
 	Cluster        string
 	Command        []string

diff --git a/provider/aws/task.go b/provider/aws/task.go
@@ -1,13 +1,14 @@
 package aws
 
 import (
+	"strings"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
 	"github.com/pkg/errors"
 	"github.com/stelligent/mu/common"
-	"strings"
 )
 
 type ecsTaskManager struct {
@@ -154,7 +155,6 @@ func (taskMgr *ecsTaskManager) StopTask(namespace string, environment string, ta
 }
 
 func getTaskDetail(ecsTask *ecs.Task, taskMgr *ecsTaskManager, cluster string, environment string, serviceName string) (*common.Task, error) {
-	log.Debugf(SvcGetTaskInfoLog, *ecsTask.TaskArn)
 	containers := []common.Container{}
 	if len(ecsTask.Containers) > Zero {
 		for _, container := range ecsTask.Containers {
@@ -172,6 +172,7 @@ func getTaskDetail(ecsTask *ecs.Task, taskMgr *ecsTaskManager, cluster string, e
 		Name:        strings.Split(*ecsTask.TaskArn, TaskARNSeparator)[1],
 		Environment: environment,
 		Service:     serviceName,
+		Status:      aws.StringValue(ecsTask.LastStatus),
 		Containers:  containers,
 	}
 	log.Debugf(SvcTaskDetailLog, task)

diff --git a/workflows/service_restarter.go b/workflows/service_restarter.go
@@ -14,13 +14,15 @@ func NewServiceRestarter(ctx *common.Context, environmentName string, serviceNam
 
 	return newPipelineExecutor(
 		workflow.serviceInput(ctx, serviceName),
-		workflow.serviceRestarter(ctx.Config.Namespace, ctx.TaskManager, environmentName, workflow.serviceName, batchSize),
+		workflow.serviceRestarter(ctx.Config.Namespace, ctx.TaskManager, environmentName, batchSize),
 	)
 }
 
-func (workflow *serviceWorkflow) serviceRestarter(namespace string, taskManager common.TaskManager, environmentName string, serviceName string, batchSize int) Executor {
+func (workflow *serviceWorkflow) serviceRestarter(namespace string, taskManager common.TaskManager, environmentName string, batchSize int) Executor {
 	return func() error {
-		tasks, err := taskManager.ListTasks(namespace, environmentName, serviceName)
+		tasks, err := taskManager.ListTasks(namespace, environmentName, workflow.serviceName)
+
+		log.Noticef("Found %v tasks for service %s in environment %s", len(tasks), workflow.serviceName, environmentName)
 
 		if err != nil {
 			return err
@@ -36,15 +38,25 @@ func (workflow *serviceWorkflow) serviceRestarter(namespace string, taskManager
 			// Polling for same length task lists
 			if (taskIdx+1)%batchSize == 0 {
 
-				newTaskList, _ := taskManager.ListTasks(namespace, environmentName, serviceName)
-				for len(newTaskList) != len(tasks) {
+				for countRunningTasks(namespace, taskManager, environmentName, workflow.serviceName) != len(tasks) {
 					duration := time.Duration(PollDelay) * time.Second
 					time.Sleep(duration)
-					newTaskList, _ = taskManager.ListTasks(namespace, environmentName, serviceName)
 				}
 			}
 		}
 
 		return nil
 	}
 }
+
+func countRunningTasks(namespace string, taskManager common.TaskManager, environmentName string, serviceName string) int {
+	newTaskList, _ := taskManager.ListTasks(namespace, environmentName, serviceName)
+	runningCount := 0
+	for _, newTask := range newTaskList {
+		if newTask.Status == "RUNNING" {
+			runningCount++
+		}
+	}
+	log.Debugf("Environment: %s, Service: %s, Running Tasks: %v", environmentName, serviceName, runningCount)
+	return runningCount
+}