worker: check cloudformation stack status

Stack update may fail due to external changes, e.g. due to removal of
a referenced security group.

This change reports certain stack statuses as problems which prevents
`last_sync_timestamp_seconds` metric update that could be
used to monitor stack update failures.

Signed-off-by: Alexander Yastrebov <alexander.yastrebov@zalando.de>

aws/cf.go

@@ -21,6 +21,7 @@ const (
 type Stack struct {
 	Name              string
 	status            string
+	statusReason      string
 	DNSName           string
 	Scheme            string
 	SecurityGroup     string
@@ -69,6 +70,29 @@ func (s *Stack) ShouldDelete() bool {
 	return true
 }
 
+// Err returns nil or an error describing the stack state.
+func (s *Stack) Err() error {
+	if s == nil {
+		return nil
+	}
+
+	switch s.status {
+	case cloudformation.StackStatusCreateInProgress,
+		cloudformation.StackStatusCreateComplete,
+		cloudformation.StackStatusUpdateInProgress,
+		cloudformation.StackStatusUpdateComplete,
+		cloudformation.StackStatusUpdateCompleteCleanupInProgress,
+		cloudformation.StackStatusDeleteInProgress,
+		cloudformation.StackStatusDeleteComplete:
+		return nil
+	}
+
+	if s.statusReason != "" {
+		return fmt.Errorf("unexpected status %s: %s", s.status, s.statusReason)
+	}
+	return fmt.Errorf("unexpected status %s", s.status)
+}
+
 type stackOutput map[string]string
 
 func newStackOutput(outputs []*cloudformation.Output) stackOutput {
@@ -472,6 +496,7 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack {
 		tags:              tags,
 		OwnerIngress:      ownerIngress,
 		status:            aws.StringValue(stack.StackStatus),
+		statusReason:      aws.StringValue(stack.StackStatusReason),
 		CWAlarmConfigHash: tags[cwAlarmConfigHashTag],
 		WAFWebACLID:       parameters[parameterLoadBalancerWAFWebACLIDParameter],
 	}

aws/cf_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestCreatingStack(t *testing.T) {
@@ -260,6 +261,45 @@ func TestIsComplete(t *testing.T) {
 
 }
 
+func TestErr(t *testing.T) {
+	const NONE = ""
+	for _, ti := range []struct {
+		stack         *Stack
+		expectedError string
+	}{
+		{stack: nil, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusCreateInProgress}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusCreateComplete}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusUpdateInProgress}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusUpdateComplete}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusUpdateCompleteCleanupInProgress}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusDeleteInProgress}, expectedError: NONE},
+		{stack: &Stack{status: cloudformation.StackStatusDeleteComplete}, expectedError: NONE},
+		//
+		{stack: &Stack{status: cloudformation.StackStatusUpdateRollbackComplete}, expectedError: "unexpected status UPDATE_ROLLBACK_COMPLETE"},
+		{
+			stack: &Stack{
+				status:       cloudformation.StackStatusUpdateRollbackInProgress,
+				statusReason: "Parameter validation failed: parameter value sg-xxx for parameter name LoadBalancerSecurityGroupParameter does not exist",
+			},
+			expectedError: "unexpected status UPDATE_ROLLBACK_IN_PROGRESS: Parameter validation failed: parameter value sg-xxx for parameter name LoadBalancerSecurityGroupParameter does not exist",
+		},
+	} {
+		testName := "nil stack"
+		if ti.stack != nil {
+			testName = ti.stack.status
+		}
+		t.Run(testName, func(t *testing.T) {
+			err := ti.stack.Err()
+			if ti.expectedError == NONE {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, ti.expectedError)
+			}
+		})
+	}
+}
+
 func TestManagementAssertion(t *testing.T) {
 	for _, ti := range []struct {
 		name  string

worker.go

@@ -282,6 +282,12 @@ func doWork(
 		return problems.Add("failed to list managed stacks: %w", err)
 	}
 
+	for _, stack := range stacks {
+		if err := stack.Err(); err != nil {
+			problems.Add("stack %s error: %w", stack.Name, err)
+		}
+	}
+
 	err = awsAdapter.UpdateAutoScalingGroupsAndInstances()
 	if err != nil {
 		return problems.Add("failed to get instances from EC2: %w", err)