elb.go

// Package elb implements ELB utilities.
package elb

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/elbv2"
	"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
	"go.uber.org/zap"
)

// DeleteELBv2 deletes all resources associated
// with the load balancer.
// TODO: is there a better way to clean up resources?
// ref. https://github.com/aws/aws-k8s-tester/issues/70
func DeleteELBv2(lg *zap.Logger, elb2API elbv2iface.ELBV2API, arn string, vpcID string, tags map[string]string) (err error) {
	recurse := false
	if arn == "" {
		lg.Info("load balancer ARN not given, querying by VPC and cluster name",
			zap.String("vpc-id", vpcID),
			zap.String("tags", fmt.Sprintf("%+v", tags)),
		)
		elbARNs := make([]string, 0)
		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
		err = elb2API.DescribeLoadBalancersPagesWithContext(
			ctx,
			&elbv2.DescribeLoadBalancersInput{},
			func(output *elbv2.DescribeLoadBalancersOutput, _ bool) bool {
				for _, ev := range output.LoadBalancers {
					arn := aws.StringValue(ev.LoadBalancerArn)
					vpcID := aws.StringValue(ev.VpcId)
					if vpcID == vpcID {
						lg.Warn("found ELBv2 for this VPC",
							zap.String("vpc-id", vpcID),
							zap.String("elb-arn", arn),
						)
						elbARNs = append(elbARNs, arn)
					} else {
						lg.Info("found ELBv2 for other VPCs", zap.String("vpc-id", vpcID), zap.String("elb-arn", arn))
					}
				}
				return true
			})
		cancel()
		if err != nil {
			lg.Warn("failed to describe ELBv2", zap.Error(err))
			return errors.New("empty ELB ARN")
		}

		lg.Info("describing tags for elb", zap.Strings("elb-arns", elbARNs))
		tout, err := elb2API.DescribeTags(&elbv2.DescribeTagsInput{ResourceArns: aws.StringSlice(elbARNs)})
		if err != nil {
			lg.Warn("failed to describe tags", zap.Error(err))
		}
		matchingARNs := make([]string, 0)
		for _, desc := range tout.TagDescriptions {
			copied := make(map[string]string)
			for k, v := range tags {
				copied[k] = v
			}
			elbARN := aws.StringValue(desc.ResourceArn)
			for _, tv := range desc.Tags {
				k, v1 := aws.StringValue(tv.Key), aws.StringValue(tv.Value)
				lg.Info("found tag", zap.String("elb-arn", elbARN), zap.String("key", k), zap.String("value", v1))
				if v2, ok := copied[k]; ok && v2 == v1 {
					delete(copied, k)
					lg.Info("found matching tag", zap.String("key", k), zap.String("value", v1))
				}
			}
			if len(copied) == 0 {
				lg.Info("found elb with matching tags; deleting", zap.String("elb-arn", elbARN))
				matchingARNs = append(matchingARNs, elbARN)
			}
		}

		lg.Info("matching elb", zap.Strings("elb-arns", matchingARNs))
		switch {
		case len(matchingARNs) == 0:
			lg.Warn("could not found matching elb")
			return errors.New("empty ELB ARN")
		case len(matchingARNs) == 1:
			lg.Warn("found 1 matching elb")
			arn = matchingARNs[0]
		case len(matchingARNs) > 1:
			lg.Warn("found multiple matching elb")
			arn = matchingARNs[0]
			recurse = true
			lg.Info("need recursively delete elb", zap.Strings("elb-arn", matchingARNs[1:]))
		}
	}

	// deleteInOrder deletes listeners, target groups, and ELB in order.
	if err = deleteInOrder(lg, elb2API, arn); err == nil {
		lg.Info("successfully deleted ELB in order")
	}
	// deleteInReverseOrder deletes ELB and expects ENIs to be detached and deleted.
	if err = deleteInReverseOrder(lg, elb2API, arn); err == nil {
		lg.Info("successfully deleted ELB in reverse order")
	}
	if recurse {
		// pass empty, so it can fetch recursively len(elbs) > 2
		return DeleteELBv2(lg, elb2API, "", vpcID, tags)
	}
	return nil
}

// deleteInOrder deletes listeners, target groups, and ELB in order.
func deleteInOrder(lg *zap.Logger, elb2API elbv2iface.ELBV2API, arn string) error {
	lbDeleted := false
	// delete listener first
	// e.g. ResourceInUse: Target group is currently in use by a listener or a rule
	lg.Info("describing listeners", zap.String("arn", arn))
	ls, err := elb2API.DescribeListeners(&elbv2.DescribeListenersInput{
		LoadBalancerArn: aws.String(arn),
	})
	if err != nil {
		if !isDeleted(err) {
			return err
		}
		lbDeleted = true
		lg.Info("load balancer has already been deleted", zap.Error(err))
	}
	if !lbDeleted && ls != nil && len(ls.Listeners) > 0 {
		for _, lv := range ls.Listeners {
			arn := aws.StringValue(lv.ListenerArn)

			lg.Info("describing rules for listener", zap.String("listener-arn", arn))
			ro, err := elb2API.DescribeRules(&elbv2.DescribeRulesInput{
				ListenerArn: lv.ListenerArn,
			})
			if err != nil {
				lg.Warn("failed to describe rules for listener", zap.Error(err))
			} else {
				for _, rv := range ro.Rules {
					ruleArn := aws.StringValue(rv.RuleArn)
					lg.Info("deleting rule for listener", zap.String("rule-arn", ruleArn))
					_, err = elb2API.DeleteRule(&elbv2.DeleteRuleInput{
						RuleArn: rv.RuleArn,
					})
					if err != nil {
						lg.Info("failed to delete rule for listener", zap.String("rule-arn", ruleArn), zap.Error(err))
					} else {
						lg.Info("deleted rule for listener",
							zap.String("listener-arn", arn),
							zap.String("rule-arn", ruleArn),
						)
					}
				}
			}

			lg.Info("deleting listener", zap.String("listener-arn", arn))
			_, err = elb2API.DeleteListener(&elbv2.DeleteListenerInput{
				ListenerArn: lv.ListenerArn,
			})
			if err != nil {
				lg.Warn("failed to delete listener", zap.String("listener-arn", arn), zap.Error(err))
			} else {
				lg.Info("deleted listener", zap.String("listener-arn", arn))
			}
		}
	}

	lg.Info("describing target groups", zap.String("arn", arn))
	to, err := elb2API.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
		LoadBalancerArn: aws.String(arn),
	})
	if err != nil {
		if !isDeleted(err) {
			return err
		}
		lbDeleted = true
		lg.Info("load balancer has already been deleted", zap.Error(err))
	}
	if !lbDeleted && to != nil && len(to.TargetGroups) > 0 {
		for _, tv := range to.TargetGroups {
			arn := aws.StringValue(tv.TargetGroupArn)
			name := aws.StringValue(tv.TargetGroupName)
			tp := aws.StringValue(tv.TargetType)
			lg.Info("deleting target group",
				zap.String("arn", arn),
				zap.String("name", name),
				zap.String("type", tp),
			)
			_, err = elb2API.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{
				TargetGroupArn: tv.TargetGroupArn,
			})
			if err != nil {
				lg.Warn("failed to delete target group", zap.Error(err))
			} else {
				lg.Info("deleted target group")
			}
		}
	}

	err = nil
	if !lbDeleted {
		for i := 0; i < 5; i++ {
			time.Sleep(10 * time.Second)
			lg.Info("deleting ELB in order", zap.String("arn", arn))
			_, err = elb2API.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{
				LoadBalancerArn: aws.String(arn),
			})
			if err == nil {
				lg.Info("successfully deleted ELB in order")
				lbDeleted = true
				break
			}
			if isDeleted(err) {
				err, lbDeleted = nil, true
				lg.Info("ELB has already been deleted", zap.Error(err))
				break
			}
			lg.Warn("failing to delete ELB in order", zap.Error(err))
		}
	}
	if err == nil && lbDeleted {
		lg.Info("deleted ELB in order")
		return nil
	}
	if err != nil && lbDeleted {
		return nil
	}

	if err != nil && !lbDeleted {
		lg.Warn("failed to delete ELB in order", zap.Error(err))
	}
	return err
}

// deleteInReverseOrder deletes ELB and expects ENIs to be detached and deleted.
func deleteInReverseOrder(lg *zap.Logger, elb2API elbv2iface.ELBV2API, arn string) error {
	var err error
	for i := 0; i < 5; i++ {
		time.Sleep(10 * time.Second)
		lg.Info("deleting ELB in reverse order", zap.String("arn", arn))
		_, err = elb2API.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{
			LoadBalancerArn: aws.String(arn),
		})
		if err == nil {
			lg.Info("successfully deleted ELB in reverse order")
			lg.Info("waiting for ENI clean up after ELB deletion")
			time.Sleep(30 * time.Second)
			break
		}
		if isDeleted(err) {
			lg.Info("ELB has already been deleted in order", zap.Error(err))
			break
		}
		lg.Warn("failing to delete ELB in order", zap.Error(err))
	}
	return err
}

func isDeleted(err error) bool {
	if err == nil {
		return false
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	return awsErr.Code() == "LoadBalancerNotFound"
}