Tracking errors and properly exiting with error code (#37)

Co-authored-by: Josh Schell <joshua.schell@disneystreaming.com>

cmd/run.go

@@ -114,12 +114,18 @@ func runCommand(cmd *cobra.Command, args []string) {
 	// Output our results
 	log.Infof(resultFormat, "Instance ID", "Region", "Profile", "Status")
 	for _, v := range output.InvocationResults {
-		switch v.Status {
-		case "Success":
-			log.Infof(resultFormat, *v.InvocationResult.InstanceId, v.Region, v.ProfileName, *v.InvocationResult.StatusDetails)
+
+		if v.Status == invocation.ClientError {
+			log.Errorf(resultFormat, "---", v.Region, v.ProfileName, v.Status)
+			failedCounter++
+			continue
+		}
+
+		if v.Status == invocation.CommandSuccess {
+			log.Infof(resultFormat, *v.InvocationResult.InstanceId, v.Region, v.ProfileName, v.Status)
 			successCounter++
-		default:
-			log.Errorf(resultFormat, *v.InvocationResult.InstanceId, v.Region, v.ProfileName, *v.InvocationResult.StatusDetails)
+		} else {
+			log.Errorf(resultFormat, *v.InvocationResult.InstanceId, v.Region, v.ProfileName, v.Status)
 			failedCounter++
 		}
 
@@ -130,7 +136,7 @@ func runCommand(cmd *cobra.Command, args []string) {
 
 		// stderr is written back at warn if the invocation was successful, and error if not
 		if *v.InvocationResult.StandardErrorContent != "" {
-			if v.Status == "Success" {
+			if v.Status == invocation.CommandSuccess {
 				log.Warn(*v.InvocationResult.StandardErrorContent)
 			} else {
 				log.Error(*v.InvocationResult.StandardErrorContent)

ssm/helpers.go

@@ -79,7 +79,8 @@ func RunInvocations(sess *session.Session, client ssmiface.SSMAPI, wg *sync.Wait
 
 	// Send our command input to SSM
 	if scOutput, err = client.SendCommand(input); err != nil {
-		sess.Logger.Errorf("Error when calling the SendCommand API for account %v in %v\n%v", sess.ProfileName, *sess.Session.Config.Region, err)
+		sess.Logger.Error("Error when calling the SendCommand API")
+		addError(results, sess, err)
 		return
 	}
 
@@ -89,7 +90,7 @@ func RunInvocations(sess *session.Session, client ssmiface.SSMAPI, wg *sync.Wait
 	// Watch status of invocation to see when it's done and we can get the output
 	for done := false; !done; time.Sleep(2 * time.Second) {
 		if done, err = checkInvocationStatus(client, commandID); err != nil {
-			sess.Logger.Error(err)
+			addError(results, sess, err)
 			return
 		}
 	}
@@ -114,7 +115,7 @@ func RunInvocations(sess *session.Session, client ssmiface.SSMAPI, wg *sync.Wait
 				case result := <-oc:
 					addInvocationResults(results, sess, result)
 				case err := <-ec:
-					sess.Logger.Error(err)
+					addError(results, sess, err)
 				}
 			}
 
@@ -126,28 +127,36 @@ func RunInvocations(sess *session.Session, client ssmiface.SSMAPI, wg *sync.Wait
 			lciInput.SetNextToken(*page.NextToken)
 			return true
 		}); err != nil {
-		sess.Logger.Error(fmt.Errorf("Error when calling ListCommandInvocations API\n%v", err))
+		sess.Logger.Error("Error when calling ListCommandInvocations API")
+		addError(results, sess, err)
 	}
-
 }
 
 func addInvocationResults(results *invocation.ResultSafe, session *session.Session, info ...*ssm.GetCommandInvocationOutput) {
 	var newResults []*invocation.Result
 	for _, v := range info {
-		var result = &invocation.Result{
+		results.Add(&invocation.Result{
 			InvocationResult: v,
 			ProfileName:      session.ProfileName,
 			Region:           *session.Session.Config.Region,
-			Status:           *v.StatusDetails,
-		}
-		newResults = append(newResults, result)
+			Status:           invocation.Status(*v.StatusDetails),
+		})
 	}
 
 	results.Lock()
 	results.InvocationResults = append(results.InvocationResults, newResults...)
 	results.Unlock()
 }
 
+func addError(results *invocation.ResultSafe, session *session.Session, err error) {
+	results.Add(&invocation.Result{
+		ProfileName: session.ProfileName,
+		Region:      *session.Session.Config.Region,
+		Status:      invocation.ClientError,
+		Error:       err,
+	})
+}
+
 // CheckInstanceReadiness iterates through a list of instances and verifies whether or not it is start-session capable. If it is, it appends the instance info to an instances.InstanceInfoSafe slice.
 func CheckInstanceReadiness(session *session.Session, client ssmiface.SSMAPI, instanceList []*ssm.InstanceInformation, limit int, readyInstancePool *instance.InstanceInfoSafe) {
 	var readyInstances, ec2Instances []*string

ssm/invocation/runner.go

@@ -52,5 +52,4 @@ func GetResult(client ssmiface.SSMAPI, commandID *string, instanceID *string, gc
 	case status != nil:
 		gci <- status
 	}
-
 }

ssm/invocation/types.go

@@ -15,16 +15,59 @@ type CommandOutputSafe struct {
 	Output []*ssm.SendCommandOutput
 }
 
+// Status ...
+type Status string
+
+const (
+	// CommandSuccess indicates ssm successfully ran a command on target(s)
+	CommandSuccess Status = "Success"
+
+	// CommandFailed indicates ssm ran a command on target(s) that failed
+	CommandFailed Status = "Failed"
+
+	// CommandPending indicates ssm ran a command on target(s) that is pending
+	CommandPending Status = "Pending"
+
+	// CommandInProgress The command has been sent to the instance but has not reached a terminal state.
+	CommandInProgress Status = "In Progress"
+
+	// CommandDeliveryTimedOut The command was not delivered to the instance before the delivery timeout expired. Delivery timeouts do not count against the parent command's MaxErrors limit, but they do contribute to whether the parent command status is Success or Incomplete. This is a terminal state.
+	CommandDeliveryTimedOut Status = "Delivery Timed Out"
+
+	// CommandExecutionTimedOut Command execution started on the instance, but the execution was not complete before the execution timeout expired. Execution timeouts count against the MaxErrors limit of the parent command. This is a terminal state.
+	CommandExecutionTimedOut Status = "Execution Timed Out"
+
+	// CommandCanceled The command was terminated before it was completed. This is a terminal state.
+	CommandCanceled Status = "Canceled"
+
+	// CommandUndeliverable The command can't be delivered to the instance. The instance might not exist or might not be responding. Undeliverable invocations don't count against the parent command's MaxErrors limit and don't contribute to whether the parent command status is Success or Incomplete. This is a terminal state.
+	CommandUndeliverable Status = "Undeliverable"
+
+	// CommandTerminated The parent command exceeded its MaxErrors limit and subsequent command invocations were canceled by the system. This is a terminal state.
+	CommandTerminated Status = "Terminated"
+
+	// ClientError indicates error occured prior to or when invoking an ssm api method
+	ClientError Status = "ClientError"
+)
+
 // Result is used to store information about an invocation run on a particular instance
 type Result struct {
 	InvocationResult *ssm.GetCommandInvocationOutput
 	ProfileName      string
 	Region           string
-	Status           string
+	Status           Status
+	Error            error
 }
 
 // ResultSafe allows for concurrent-safe access to a slice of InvocationResult info
 type ResultSafe struct {
 	sync.Mutex
 	InvocationResults []*Result
 }
+
+// Add allows appending results safely
+func (results *ResultSafe) Add(result *Result) {
+	results.Lock()
+	results.InvocationResults = append(results.InvocationResults, result)
+	results.Unlock()
+}