From d9fa618de696a677c12c03d29110e260c5857188 Mon Sep 17 00:00:00 2001 From: andrewsykim Date: Sat, 2 Sep 2017 17:23:25 -0400 Subject: [PATCH] refactor resource tracker to be usable across packages --- cmd/kops/delete_cluster.go | 11 +- hack/.packages | 1 + pkg/resources/aws.go | 367 ++++++++++++------------ pkg/resources/aws_test.go | 5 +- pkg/resources/cluster_resources.go | 54 ++-- pkg/resources/digitalocean/resources.go | 37 +++ pkg/resources/do.go | 14 +- pkg/resources/gce.go | 185 ++++++------ pkg/resources/tracker/tracker.go | 42 +++ pkg/resources/vsphere.go | 27 +- 10 files changed, 409 insertions(+), 334 deletions(-) create mode 100644 pkg/resources/digitalocean/resources.go create mode 100644 pkg/resources/tracker/tracker.go diff --git a/cmd/kops/delete_cluster.go b/cmd/kops/delete_cluster.go index 37e57b5818d48..e7078ec4b026f 100644 --- a/cmd/kops/delete_cluster.go +++ b/cmd/kops/delete_cluster.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/resources" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -148,7 +149,7 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti return err } - clusterResources := make(map[string]*resources.ResourceTracker) + clusterResources := make(map[string]*tracker.Resource) for k, resource := range allResources { if resource.Shared { continue @@ -162,16 +163,16 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti wouldDeleteCloudResources = true t := &tables.Table{} - t.AddColumn("TYPE", func(r *resources.ResourceTracker) string { + t.AddColumn("TYPE", func(r *tracker.Resource) string { return r.Type }) - t.AddColumn("ID", func(r *resources.ResourceTracker) string { + t.AddColumn("ID", func(r *tracker.Resource) string { return r.ID }) - t.AddColumn("NAME", func(r *resources.ResourceTracker) string { + t.AddColumn("NAME", func(r *tracker.Resource) string { return r.Name }) - var l []*resources.ResourceTracker + var l []*tracker.Resource for _, v := range clusterResources { l = append(l, v) } diff --git a/hack/.packages b/hack/.packages index 067499f5a7f13..69cdd86f7d1fe 100644 --- a/hack/.packages +++ b/hack/.packages @@ -77,6 +77,7 @@ k8s.io/kops/pkg/pretty k8s.io/kops/pkg/resources k8s.io/kops/pkg/resources/digitalocean k8s.io/kops/pkg/resources/digitalocean/dns +k8s.io/kops/pkg/resources/tracker k8s.io/kops/pkg/systemd k8s.io/kops/pkg/templates k8s.io/kops/pkg/testutils diff --git a/pkg/resources/aws.go b/pkg/resources/aws.go index 99576172531da..c2a3b4008fbef 100644 --- a/pkg/resources/aws.go +++ b/pkg/resources/aws.go @@ -32,6 +32,7 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ) @@ -43,12 +44,12 @@ const ( TypeLoadBalancer = "load-balancer" ) -type listFn func(fi.Cloud, string) ([]*ResourceTracker, error) +type listFn func(fi.Cloud, string) ([]*tracker.Resource, error) -func (c *ClusterResources) listResourcesAWS() (map[string]*ResourceTracker, error) { +func (c *ClusterResources) listResourcesAWS() (map[string]*tracker.Resource, error) { cloud := c.Cloud.(awsup.AWSCloud) - resources := make(map[string]*ResourceTracker) + resources := make(map[string]*tracker.Resource) // These are the functions that are used for looking up // cluster resources by their tags. @@ -80,11 +81,11 @@ func (c *ClusterResources) listResourcesAWS() (map[string]*ResourceTracker, erro ListIAMRoles, } for _, fn := range listFunctions { - trackers, err := fn(cloud, c.ClusterName) + resourceTrackers, err := fn(cloud, c.ClusterName) if err != nil { return nil, err } - for _, t := range trackers { + for _, t := range resourceTrackers { resources[t.Type+":"+t.ID] = t } } @@ -108,11 +109,11 @@ func (c *ClusterResources) listResourcesAWS() (map[string]*ResourceTracker, erro } vpc := resources["vpc:"+vpcID] if vpc != nil && resources["internet-gateway:"+igwID] == nil { - resources["internet-gateway:"+igwID] = &ResourceTracker{ + resources["internet-gateway:"+igwID] = &tracker.Resource{ Name: FindName(igw.Tags), ID: igwID, Type: "internet-gateway", - deleter: DeleteInternetGateway, + Deleter: DeleteInternetGateway, Shared: vpc.Shared, // Shared iff the VPC is shared } } @@ -146,7 +147,7 @@ func (c *ClusterResources) listResourcesAWS() (map[string]*ResourceTracker, erro { // We delete a NAT gateway if it is linked to our route table - routeTableIds := make(map[string]*ResourceTracker) + routeTableIds := make(map[string]*tracker.Resource) for _, resource := range resources { if resource.Type != ec2.ResourceTypeRouteTable { continue @@ -165,7 +166,7 @@ func (c *ClusterResources) listResourcesAWS() (map[string]*ResourceTracker, erro } for k, t := range resources { - if t.done { + if t.Done { delete(resources, k) } } @@ -184,7 +185,7 @@ func BuildEC2Filters(cloud fi.Cloud) []*ec2.Filter { return filters } -func addUntaggedRouteTables(cloud awsup.AWSCloud, clusterName string, resources map[string]*ResourceTracker) error { +func addUntaggedRouteTables(cloud awsup.AWSCloud, clusterName string, resources map[string]*tracker.Resource) error { // We sometimes have trouble tagging the route table (eventual consistency, e.g. #597) // If we are deleting the VPC, we should delete the route table // (no real reason not to; easy to recreate; no real state etc) @@ -364,7 +365,7 @@ func matchesElbTags(tags map[string]string, actual []*elb.Tag) bool { // Delete(cloud fi.Cloud) error //} -func DeleteInstance(cloud fi.Cloud, t *ResourceTracker) error { +func DeleteInstance(cloud fi.Cloud, t *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := t.ID @@ -383,7 +384,7 @@ func DeleteInstance(cloud fi.Cloud, t *ResourceTracker) error { return nil } -func DeleteCloudFormationStack(cloud fi.Cloud, t *ResourceTracker) error { +func DeleteCloudFormationStack(cloud fi.Cloud, t *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := t.ID @@ -399,16 +400,16 @@ func DeleteCloudFormationStack(cloud fi.Cloud, t *ResourceTracker) error { return nil } -func DumpCloudFormationStack(r *ResourceTracker) (interface{}, error) { +func DumpCloudFormationStack(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = r.Type - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } -func ListCloudFormationStacks(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { - var trackers []*ResourceTracker +func ListCloudFormationStacks(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { + var resourceTrackers []*tracker.Resource request := &cloudformation.ListStacksInput{} c := cloud.(awsup.AWSCloud) response, err := c.CloudFormation().ListStacks(request) @@ -417,22 +418,22 @@ func ListCloudFormationStacks(cloud fi.Cloud, clusterName string) ([]*ResourceTr } for _, stack := range response.StackSummaries { if *stack.StackName == clusterName { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: *stack.StackName, ID: *stack.StackId, Type: "cloud-formation", - deleter: DeleteCloudFormationStack, + Deleter: DeleteCloudFormationStack, Dumper: DumpCloudFormationStack, - obj: stack, + Obj: stack, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } - return trackers, nil + return resourceTrackers, nil } -func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListInstances(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) glog.V(2).Infof("Querying EC2 instances") @@ -440,7 +441,7 @@ func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, erro Filters: BuildEC2Filters(cloud), } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource err := c.EC2().DescribeInstancesPages(request, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { for _, reservation := range p.Reservations { @@ -462,13 +463,13 @@ func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, erro } } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(instance.Tags), ID: id, Type: ec2.ResourceTypeInstance, - deleter: DeleteInstance, + Deleter: DeleteInstance, Dumper: DumpInstance, - obj: instance, + Obj: instance, } var blocks []string @@ -486,9 +487,9 @@ func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, erro blocks = append(blocks, "subnet:"+aws.StringValue(instance.SubnetId)) blocks = append(blocks, "vpc:"+aws.StringValue(instance.VpcId)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } @@ -498,18 +499,18 @@ func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, erro return nil, fmt.Errorf("error describing instances: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func DumpInstance(r *ResourceTracker) (interface{}, error) { +func DumpInstance(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = ec2.ResourceTypeInstance - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } -func DeleteSecurityGroup(cloud fi.Cloud, t *ResourceTracker) error { +func DeleteSecurityGroup(cloud fi.Cloud, t *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := t.ID @@ -564,41 +565,41 @@ func DeleteSecurityGroup(cloud fi.Cloud, t *ResourceTracker) error { return nil } -func DumpSecurityGroup(r *ResourceTracker) (interface{}, error) { +func DumpSecurityGroup(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = ec2.ResourceTypeSecurityGroup - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } -func ListSecurityGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListSecurityGroups(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { groups, err := DescribeSecurityGroups(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, sg := range groups { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(sg.Tags), ID: aws.StringValue(sg.GroupId), Type: "security-group", - deleter: DeleteSecurityGroup, + Deleter: DeleteSecurityGroup, Dumper: DumpSecurityGroup, - obj: sg, + Obj: sg, } var blocks []string blocks = append(blocks, "vpc:"+aws.StringValue(sg.VpcId)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func DescribeSecurityGroups(cloud fi.Cloud) ([]*ec2.SecurityGroup, error) { @@ -616,7 +617,7 @@ func DescribeSecurityGroups(cloud fi.Cloud) ([]*ec2.SecurityGroup, error) { return response.SecurityGroups, nil } -func DeleteVolume(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteVolume(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -639,32 +640,32 @@ func DeleteVolume(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListVolumes(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListVolumes(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) volumes, err := DescribeVolumes(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource elasticIPs := make(map[string]bool) for _, volume := range volumes { id := aws.StringValue(volume.VolumeId) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(volume.Tags), ID: id, Type: "volume", - deleter: DeleteVolume, + Deleter: DeleteVolume, } var blocks []string //blocks = append(blocks, "vpc:" + aws.StringValue(rt.VpcId)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) // Check for an elastic IP tag for _, tag := range volume.Tags { @@ -694,19 +695,19 @@ func ListVolumes(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: ip, ID: aws.StringValue(address.AllocationId), Type: TypeElasticIp, - deleter: DeleteElasticIP, + Deleter: DeleteElasticIP, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } - return trackers, nil + return resourceTrackers, nil } func DescribeVolumes(cloud fi.Cloud) ([]*ec2.Volume, error) { @@ -732,7 +733,7 @@ func DescribeVolumes(cloud fi.Cloud) ([]*ec2.Volume, error) { return volumes, nil } -func DeleteKeypair(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteKeypair(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) name := r.Name @@ -748,7 +749,7 @@ func DeleteKeypair(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListKeypairs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListKeypairs(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { if !strings.Contains(clusterName, ".") { glog.Infof("cluster %q is legacy (kube-up) cluster; won't delete keypairs", clusterName) return nil, nil @@ -768,24 +769,24 @@ func ListKeypairs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error return nil, fmt.Errorf("error listing KeyPairs: %v", err) } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, keypair := range response.KeyPairs { name := aws.StringValue(keypair.KeyName) if name != keypairName && !strings.HasPrefix(name, keypairName+"-") { continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: name, ID: name, Type: "keypair", - deleter: DeleteKeypair, + Deleter: DeleteKeypair, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func IsDependencyViolation(err error) bool { @@ -801,7 +802,7 @@ func IsDependencyViolation(err error) bool { } } -func DeleteSubnet(cloud fi.Cloud, tracker *ResourceTracker) error { +func DeleteSubnet(cloud fi.Cloud, tracker *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := tracker.ID @@ -823,14 +824,14 @@ func DeleteSubnet(cloud fi.Cloud, tracker *ResourceTracker) error { return nil } -func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListSubnets(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) subnets, err := DescribeSubnets(cloud) if err != nil { return nil, fmt.Errorf("error listing subnets: %v", err) } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource elasticIPs := sets.NewString() ownedElasticIPs := sets.NewString() natGatewayIds := sets.NewString() @@ -839,15 +840,15 @@ func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) subnetID := aws.StringValue(subnet.SubnetId) shared := HasSharedTag("subnet:"+subnetID, subnet.Tags, clusterName) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(subnet.Tags), ID: subnetID, Type: "subnet", - deleter: DeleteSubnet, + Deleter: DeleteSubnet, Shared: shared, } - tracker.blocks = append(tracker.blocks, "vpc:"+aws.StringValue(subnet.VpcId)) - trackers = append(trackers, tracker) + resourceTracker.Blocks = append(resourceTracker.Blocks, "vpc:"+aws.StringValue(subnet.VpcId)) + resourceTrackers = append(resourceTrackers, resourceTracker) // Get tags and append with EIPs/NGWs as needed for _, tag := range subnet.Tags { @@ -890,14 +891,14 @@ func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: ip, ID: aws.StringValue(address.AllocationId), Type: TypeElasticIp, - deleter: DeleteElasticIP, + Deleter: DeleteElasticIP, Shared: !ownedElasticIPs.Has(ip), } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } @@ -940,26 +941,26 @@ func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: id, ID: id, Type: TypeNatGateway, - deleter: DeleteNatGateway, + Deleter: DeleteNatGateway, Shared: sharedNgwIds.Has(id) || !ownedNatGatewayIds.Has(id), } // The NAT gateway blocks deletion of any associated Elastic IPs for _, address := range ngw.NatGatewayAddresses { if address.AllocationId != nil { - tracker.blocks = append(tracker.blocks, TypeElasticIp+":"+aws.StringValue(address.AllocationId)) + resourceTracker.Blocks = append(resourceTracker.Blocks, TypeElasticIp+":"+aws.StringValue(address.AllocationId)) } } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } - return trackers, nil + return resourceTrackers, nil } func DescribeSubnets(cloud fi.Cloud) ([]*ec2.Subnet, error) { @@ -977,7 +978,7 @@ func DescribeSubnets(cloud fi.Cloud) ([]*ec2.Subnet, error) { return response.Subnets, nil } -func DeleteRouteTable(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteRouteTable(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1031,28 +1032,28 @@ func DescribeRouteTables(cloud fi.Cloud) ([]*ec2.RouteTable, error) { return response.RouteTables, nil } -func ListRouteTables(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListRouteTables(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { routeTables, err := DescribeRouteTables(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, rt := range routeTables { - tracker := buildTrackerForRouteTable(rt) - trackers = append(trackers, tracker) + resourceTracker := buildTrackerForRouteTable(rt) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func buildTrackerForRouteTable(rt *ec2.RouteTable) *ResourceTracker { - tracker := &ResourceTracker{ +func buildTrackerForRouteTable(rt *ec2.RouteTable) *tracker.Resource { + resourceTracker := &tracker.Resource{ Name: FindName(rt.Tags), ID: aws.StringValue(rt.RouteTableId), Type: ec2.ResourceTypeRouteTable, - deleter: DeleteRouteTable, + Deleter: DeleteRouteTable, } var blocks []string @@ -1064,13 +1065,13 @@ func buildTrackerForRouteTable(rt *ec2.RouteTable) *ResourceTracker { blocked = append(blocked, "subnet:"+aws.StringValue(a.SubnetId)) } - tracker.blocks = blocks - tracker.blocked = blocked + resourceTracker.Blocks = blocks + resourceTracker.Blocked = blocked - return tracker + return resourceTracker } -func DeleteDhcpOptions(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteDhcpOptions(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1089,30 +1090,30 @@ func DeleteDhcpOptions(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListDhcpOptions(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListDhcpOptions(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { dhcpOptions, err := DescribeDhcpOptions(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, o := range dhcpOptions { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(o.Tags), ID: aws.StringValue(o.DhcpOptionsId), Type: "dhcp-options", - deleter: DeleteDhcpOptions, + Deleter: DeleteDhcpOptions, } var blocks []string - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func DescribeDhcpOptions(cloud fi.Cloud) ([]*ec2.DhcpOptions, error) { @@ -1130,7 +1131,7 @@ func DescribeDhcpOptions(cloud fi.Cloud) ([]*ec2.DhcpOptions, error) { return response.DhcpOptions, nil } -func DeleteInternetGateway(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteInternetGateway(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1194,20 +1195,20 @@ func DeleteInternetGateway(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListInternetGateways(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListInternetGateways(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { gateways, err := DescribeInternetGateways(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, o := range gateways { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(o.Tags), ID: aws.StringValue(o.InternetGatewayId), Type: "internet-gateway", - deleter: DeleteInternetGateway, + Deleter: DeleteInternetGateway, } var blocks []string @@ -1216,12 +1217,12 @@ func ListInternetGateways(cloud fi.Cloud, clusterName string) ([]*ResourceTracke blocks = append(blocks, "vpc:"+aws.StringValue(a.VpcId)) } } - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func DescribeInternetGateways(cloud fi.Cloud) ([]*ec2.InternetGateway, error) { @@ -1266,7 +1267,7 @@ func DescribeInternetGatewaysIgnoreTags(cloud fi.Cloud) ([]*ec2.InternetGateway, return gateways, nil } -func DeleteVPC(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteVPC(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1285,11 +1286,11 @@ func DeleteVPC(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func DumpVPC(r *ResourceTracker) (interface{}, error) { +func DumpVPC(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = ec2.ResourceTypeVpc - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } @@ -1308,38 +1309,38 @@ func DescribeVPCs(cloud fi.Cloud) ([]*ec2.Vpc, error) { return response.Vpcs, nil } -func ListVPCs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListVPCs(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { vpcs, err := DescribeVPCs(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, v := range vpcs { vpcID := aws.StringValue(v.VpcId) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindName(v.Tags), ID: vpcID, Type: ec2.ResourceTypeVpc, - deleter: DeleteVPC, + Deleter: DeleteVPC, Dumper: DumpVPC, - obj: v, + Obj: v, Shared: HasSharedTag(ec2.ResourceTypeVpc+":"+vpcID, v.Tags, clusterName), } var blocks []string blocks = append(blocks, "dhcp-options:"+aws.StringValue(v.DhcpOptionsId)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func DeleteAutoScalingGroup(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteAutoScalingGroup(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1359,7 +1360,7 @@ func DeleteAutoScalingGroup(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListAutoScalingGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListAutoScalingGroups(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) tags := c.Tags() @@ -1369,14 +1370,14 @@ func ListAutoScalingGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTrack return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, asg := range asgs { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindASGName(asg.Tags), ID: aws.StringValue(asg.AutoScalingGroupName), Type: "autoscaling-group", - deleter: DeleteAutoScalingGroup, + Deleter: DeleteAutoScalingGroup, } var blocks []string @@ -1389,20 +1390,20 @@ func ListAutoScalingGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTrack } blocks = append(blocks, TypeAutoscalingLaunchConfig+":"+aws.StringValue(asg.LaunchConfigurationName)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func FindAutoScalingLaunchConfigurations(cloud fi.Cloud, securityGroups sets.String) ([]*ResourceTracker, error) { +func FindAutoScalingLaunchConfigurations(cloud fi.Cloud, securityGroups sets.String) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) glog.V(2).Infof("Finding all Autoscaling LaunchConfigurations by security group") - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource request := &autoscaling.DescribeLaunchConfigurationsInput{} err := c.Autoscaling().DescribeLaunchConfigurationsPages(request, func(p *autoscaling.DescribeLaunchConfigurationsOutput, lastPage bool) bool { @@ -1418,19 +1419,19 @@ func FindAutoScalingLaunchConfigurations(cloud fi.Cloud, securityGroups sets.Str continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: aws.StringValue(t.LaunchConfigurationName), ID: aws.StringValue(t.LaunchConfigurationName), Type: TypeAutoscalingLaunchConfig, - deleter: DeleteAutoscalingLaunchConfiguration, + Deleter: DeleteAutoscalingLaunchConfiguration, } var blocks []string //blocks = append(blocks, TypeAutoscalingLaunchConfig + ":" + aws.StringValue(asg.LaunchConfigurationName)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } return true }) @@ -1438,10 +1439,10 @@ func FindAutoScalingLaunchConfigurations(cloud fi.Cloud, securityGroups sets.Str return nil, fmt.Errorf("error listing autoscaling LaunchConfigurations: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func FindNatGateways(cloud fi.Cloud, routeTables map[string]*ResourceTracker) ([]*ResourceTracker, error) { +func FindNatGateways(cloud fi.Cloud, routeTables map[string]*tracker.Resource) ([]*tracker.Resource, error) { if len(routeTables) == 0 { return nil, nil } @@ -1488,7 +1489,7 @@ func FindNatGateways(cloud fi.Cloud, routeTables map[string]*ResourceTracker) ([ } } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource if len(natGatewayIds) != 0 { request := &ec2.DescribeNatGatewaysInput{} for natGatewayId := range natGatewayIds { @@ -1505,14 +1506,14 @@ func FindNatGateways(cloud fi.Cloud, routeTables map[string]*ResourceTracker) ([ for _, t := range response.NatGateways { natGatewayId := aws.StringValue(t.NatGatewayId) - ngwTracker := &ResourceTracker{ + ngwTracker := &tracker.Resource{ Name: natGatewayId, ID: natGatewayId, Type: TypeNatGateway, - deleter: DeleteNatGateway, + Deleter: DeleteNatGateway, Shared: !ownedNatGatewayIds.Has(natGatewayId), } - trackers = append(trackers, ngwTracker) + resourceTrackers = append(resourceTrackers, ngwTracker) // If we're deleting the NatGateway, we should delete the ElasticIP also for _, address := range t.NatGatewayAddresses { @@ -1525,22 +1526,22 @@ func FindNatGateways(cloud fi.Cloud, routeTables map[string]*ResourceTracker) ([ name = aws.StringValue(address.AllocationId) } - eipTracker := &ResourceTracker{ + eipTracker := &tracker.Resource{ Name: name, ID: aws.StringValue(address.AllocationId), Type: TypeElasticIp, - deleter: DeleteElasticIP, + Deleter: DeleteElasticIP, Shared: !ownedNatGatewayIds.Has(natGatewayId), } - trackers = append(trackers, eipTracker) + resourceTrackers = append(resourceTrackers, eipTracker) - ngwTracker.blocks = append(ngwTracker.blocks, eipTracker.Type+":"+eipTracker.ID) + ngwTracker.Blocks = append(ngwTracker.Blocks, eipTracker.Type+":"+eipTracker.ID) } } } } - return trackers, nil + return resourceTrackers, nil } // extractClusterName performs string-matching / parsing to determine the ClusterName in some instance-data @@ -1590,7 +1591,7 @@ func extractClusterName(userData string) string { return clusterName } -func DeleteAutoscalingLaunchConfiguration(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteAutoscalingLaunchConfiguration(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1605,7 +1606,7 @@ func DeleteAutoscalingLaunchConfiguration(cloud fi.Cloud, r *ResourceTracker) er return nil } -func DeleteELB(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteELB(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := r.ID @@ -1624,30 +1625,30 @@ func DeleteELB(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func DumpELB(r *ResourceTracker) (interface{}, error) { +func DumpELB(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = TypeLoadBalancer - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } -func ListELBs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListELBs(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { elbs, elbTags, err := DescribeELBs(cloud) if err != nil { return nil, err } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, elb := range elbs { id := aws.StringValue(elb.LoadBalancerName) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: FindELBName(elbTags[id]), ID: id, Type: TypeLoadBalancer, - deleter: DeleteELB, + Deleter: DeleteELB, Dumper: DumpELB, - obj: elb, + Obj: elb, } var blocks []string @@ -1659,12 +1660,12 @@ func ListELBs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { } blocks = append(blocks, "vpc:"+aws.StringValue(elb.VPCId)) - tracker.blocks = blocks + resourceTracker.Blocks = blocks - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func DescribeELBs(cloud fi.Cloud) ([]*elb.LoadBalancerDescription, map[string][]*elb.Tag, error) { @@ -1726,7 +1727,7 @@ func DescribeELBs(cloud fi.Cloud) ([]*elb.LoadBalancerDescription, map[string][] return elbs, elbTags, nil } -func DeleteElasticIP(cloud fi.Cloud, t *ResourceTracker) error { +func DeleteElasticIP(cloud fi.Cloud, t *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := t.ID @@ -1750,7 +1751,7 @@ func DeleteElasticIP(cloud fi.Cloud, t *ResourceTracker) error { return nil } -func DeleteNatGateway(cloud fi.Cloud, t *ResourceTracker) error { +func DeleteNatGateway(cloud fi.Cloud, t *tracker.Resource) error { c := cloud.(awsup.AWSCloud) id := t.ID @@ -1769,16 +1770,16 @@ func DeleteNatGateway(cloud fi.Cloud, t *ResourceTracker) error { return nil } -func deleteRoute53Records(cloud fi.Cloud, zone *route53.HostedZone, trackers []*ResourceTracker) error { +func deleteRoute53Records(cloud fi.Cloud, zone *route53.HostedZone, resourceTrackers []*tracker.Resource) error { c := cloud.(awsup.AWSCloud) var changes []*route53.Change var names []string - for _, tracker := range trackers { - names = append(names, tracker.Name) + for _, resourceTracker := range resourceTrackers { + names = append(names, resourceTracker.Name) changes = append(changes, &route53.Change{ Action: aws.String("DELETE"), - ResourceRecordSet: tracker.obj.(*route53.ResourceRecordSet), + ResourceRecordSet: resourceTracker.Obj.(*route53.ResourceRecordSet), }) } human := strings.Join(names, ", ") @@ -1798,11 +1799,11 @@ func deleteRoute53Records(cloud fi.Cloud, zone *route53.HostedZone, trackers []* return nil } -func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { - var trackers []*ResourceTracker +func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { + var resourceTrackers []*tracker.Resource if dns.IsGossipHostname(clusterName) { - return trackers, nil + return resourceTrackers, nil } c := cloud.(awsup.AWSCloud) @@ -1868,17 +1869,17 @@ func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: aws.StringValue(rrs.Name), ID: hostedZoneID + "/" + aws.StringValue(rrs.Name), Type: "route53-record", - groupKey: hostedZoneID, - groupDeleter: func(cloud fi.Cloud, trackers []*ResourceTracker) error { - return deleteRoute53Records(cloud, zone, trackers) + GroupKey: hostedZoneID, + GroupDeleter: func(cloud fi.Cloud, resourceTrackers []*tracker.Resource) error { + return deleteRoute53Records(cloud, zone, resourceTrackers) }, - obj: rrs, + Obj: rrs, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } return true }) @@ -1887,10 +1888,10 @@ func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, } } - return trackers, nil + return resourceTrackers, nil } -func DeleteIAMRole(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteIAMRole(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) roleName := r.Name @@ -1942,7 +1943,7 @@ func DeleteIAMRole(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListIAMRoles(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListIAMRoles(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) remove := make(map[string]bool) @@ -1968,26 +1969,26 @@ func ListIAMRoles(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error } } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, role := range roles { name := aws.StringValue(role.RoleName) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: name, ID: name, Type: "iam-role", - deleter: DeleteIAMRole, + Deleter: DeleteIAMRole, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func DeleteIAMInstanceProfile(cloud fi.Cloud, r *ResourceTracker) error { +func DeleteIAMInstanceProfile(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(awsup.AWSCloud) - profile := r.obj.(*iam.InstanceProfile) + profile := r.Obj.(*iam.InstanceProfile) name := aws.StringValue(profile.InstanceProfileName) // Remove roles @@ -2020,7 +2021,7 @@ func DeleteIAMInstanceProfile(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func ListIAMInstanceProfiles(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { +func ListIAMInstanceProfiles(cloud fi.Cloud, clusterName string) ([]*tracker.Resource, error) { c := cloud.(awsup.AWSCloud) remove := make(map[string]bool) @@ -2044,22 +2045,22 @@ func ListIAMInstanceProfiles(cloud fi.Cloud, clusterName string) ([]*ResourceTra return nil, fmt.Errorf("error listing IAM instance profiles: %v", err) } - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource for _, profile := range profiles { name := aws.StringValue(profile.InstanceProfileName) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: name, ID: name, Type: "iam-instance-profile", - deleter: DeleteIAMInstanceProfile, - obj: profile, + Deleter: DeleteIAMInstanceProfile, + Obj: profile, } - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } func FindName(tags []*ec2.Tag) string { diff --git a/pkg/resources/aws_test.go b/pkg/resources/aws_test.go index 100a6463bcec3..a4b78d3363b2d 100644 --- a/pkg/resources/aws_test.go +++ b/pkg/resources/aws_test.go @@ -24,12 +24,13 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "k8s.io/kops/cloudmock/aws/mockec2" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ) func TestAddUntaggedRouteTables(t *testing.T) { cloud := awsup.BuildMockAWSCloud("us-east-1", "abc") - resources := make(map[string]*ResourceTracker) + resources := make(map[string]*tracker.Resource) clusterName := "me.example.com" @@ -71,7 +72,7 @@ func TestAddUntaggedRouteTables(t *testing.T) { RouteTableId: aws.String("rt-5555"), }) - resources["vpc:vpc-1234"] = &ResourceTracker{} + resources["vpc:vpc-1234"] = &tracker.Resource{} err := addUntaggedRouteTables(cloud, clusterName, resources) if err != nil { diff --git a/pkg/resources/cluster_resources.go b/pkg/resources/cluster_resources.go index fdbd90557debb..6087cbff6e8fc 100644 --- a/pkg/resources/cluster_resources.go +++ b/pkg/resources/cluster_resources.go @@ -23,36 +23,16 @@ import ( "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi" ) -type ResourceTracker struct { - Name string - Type string - ID string - - // If true, this resource is not owned by the cluster - Shared bool - - blocks []string - blocked []string - done bool - - deleter func(cloud fi.Cloud, tracker *ResourceTracker) error - groupKey string - groupDeleter func(cloud fi.Cloud, trackers []*ResourceTracker) error - - Dumper func(r *ResourceTracker) (interface{}, error) - - obj interface{} -} - var _ Resources = &ClusterResources{} // Resources is a representation of a cluster with abilities to ListResources and DeleteResources type Resources interface { - ListResources() (map[string]*ResourceTracker, error) - DeleteResources(resources map[string]*ResourceTracker) error + ListResources() (map[string]*tracker.Resource, error) + DeleteResources(resources map[string]*tracker.Resource) error } // ClusterResources is an implementation of Resources @@ -65,7 +45,7 @@ type ClusterResources struct { Region string } -func (c *ClusterResources) ListResources() (map[string]*ResourceTracker, error) { +func (c *ClusterResources) ListResources() (map[string]*tracker.Resource, error) { switch c.Cloud.ProviderID() { case kops.CloudProviderAWS: return c.listResourcesAWS() @@ -80,23 +60,23 @@ func (c *ClusterResources) ListResources() (map[string]*ResourceTracker, error) } } -func (c *ClusterResources) DeleteResources(resources map[string]*ResourceTracker) error { +func (c *ClusterResources) DeleteResources(resources map[string]*tracker.Resource) error { depMap := make(map[string][]string) - done := make(map[string]*ResourceTracker) + done := make(map[string]*tracker.Resource) var mutex sync.Mutex for k, t := range resources { - for _, block := range t.blocks { + for _, block := range t.Blocks { depMap[block] = append(depMap[block], k) } - for _, blocked := range t.blocked { + for _, blocked := range t.Blocked { depMap[k] = append(depMap[k], blocked) } - if t.done { + if t.Done { done[k] = t } } @@ -110,10 +90,10 @@ func (c *ClusterResources) DeleteResources(resources map[string]*ResourceTracker for { // TODO: Some form of default ordering based on types? - failed := make(map[string]*ResourceTracker) + failed := make(map[string]*tracker.Resource) for { - phase := make(map[string]*ResourceTracker) + phase := make(map[string]*tracker.Resource) for k, r := range resources { if _, d := done[k]; d { @@ -143,9 +123,9 @@ func (c *ClusterResources) DeleteResources(resources map[string]*ResourceTracker break } - groups := make(map[string][]*ResourceTracker) + groups := make(map[string][]*tracker.Resource) for k, t := range phase { - groupKey := t.groupKey + groupKey := t.GroupKey if groupKey == "" { groupKey = "_" + k } @@ -156,7 +136,7 @@ func (c *ClusterResources) DeleteResources(resources map[string]*ResourceTracker for _, trackers := range groups { wg.Add(1) - go func(trackers []*ResourceTracker) { + go func(trackers []*tracker.Resource) { mutex.Lock() for _, t := range trackers { k := t.Type + ":" + t.ID @@ -169,13 +149,13 @@ func (c *ClusterResources) DeleteResources(resources map[string]*ResourceTracker human := trackers[0].Type + ":" + trackers[0].ID var err error - if trackers[0].groupDeleter != nil { - err = trackers[0].groupDeleter(c.Cloud, trackers) + if trackers[0].GroupDeleter != nil { + err = trackers[0].GroupDeleter(c.Cloud, trackers) } else { if len(trackers) != 1 { glog.Fatalf("found group without groupKey") } - err = trackers[0].deleter(c.Cloud, trackers[0]) + err = trackers[0].Deleter(c.Cloud, trackers[0]) } if err != nil { mutex.Lock() diff --git a/pkg/resources/digitalocean/resources.go b/pkg/resources/digitalocean/resources.go new file mode 100644 index 0000000000000..5589c41039e5a --- /dev/null +++ b/pkg/resources/digitalocean/resources.go @@ -0,0 +1,37 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digitalocean + +import ( + "k8s.io/kops/pkg/resources/tracker" + "k8s.io/kops/upup/pkg/fi" +) + +type Resources struct { + Cloud fi.Cloud + ClusterName string +} + +// ListResources fetches all digitalocean resources into tracker.Resources +func (r *Resources) ListResources() (map[string]*tracker.Resource, error) { + return nil, nil +} + +// DeleteResources deletes all resources passed in the form in tracker.Resources +func (r *Resources) DeleteResources(resources map[string]*tracker.Resource) error { + return nil +} diff --git a/pkg/resources/do.go b/pkg/resources/do.go index 8c7a3143b09db..2fd52c934801e 100644 --- a/pkg/resources/do.go +++ b/pkg/resources/do.go @@ -16,6 +16,16 @@ limitations under the License. package resources -func (c *ClusterResources) listResourcesDO() (map[string]*ResourceTracker, error) { - return nil, nil +import ( + "k8s.io/kops/pkg/resources/digitalocean" + "k8s.io/kops/pkg/resources/tracker" +) + +func (c *ClusterResources) listResourcesDO() (map[string]*tracker.Resource, error) { + r := digitalocean.Resources{ + Cloud: c.Cloud, + ClusterName: c.ClusterName, + } + + return r.ListResources() } diff --git a/pkg/resources/gce.go b/pkg/resources/gce.go index f2351dcf851ac..d5ae55a0cb118 100644 --- a/pkg/resources/gce.go +++ b/pkg/resources/gce.go @@ -26,12 +26,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kubernetes/federation/pkg/dnsprovider" ) -type gceListFn func() ([]*ResourceTracker, error) +type gceListFn func() ([]*tracker.Resource, error) const ( typeInstance = "Instance" @@ -44,13 +45,13 @@ const ( typeRoute = "Route" ) -func (c *ClusterResources) listResourcesGCE() (map[string]*ResourceTracker, error) { +func (c *ClusterResources) listResourcesGCE() (map[string]*tracker.Resource, error) { gceCloud := c.Cloud.(gce.GCECloud) if c.Region == "" { c.Region = gceCloud.Region() } - resources := make(map[string]*ResourceTracker) + resources := make(map[string]*tracker.Resource) d := &clusterDiscoveryGCE{ cloud: c.Cloud, @@ -91,11 +92,11 @@ func (c *ClusterResources) listResourcesGCE() (map[string]*ResourceTracker, erro d.listAddresses, } for _, fn := range listFunctions { - trackers, err := fn() + resourceTrackers, err := fn() if err != nil { return nil, err } - for _, t := range trackers { + for _, t := range resourceTrackers { resources[t.Type+":"+t.ID] = t } } @@ -104,17 +105,17 @@ func (c *ClusterResources) listResourcesGCE() (map[string]*ResourceTracker, erro // Technically we still have a race condition here - until the master(s) are terminated, they will keep // creating routes. Another option might be to have a post-destroy cleanup, and only remove routes with no target. { - trackers, err := d.listRoutes(resources) + resourceTrackers, err := d.listRoutes(resources) if err != nil { return nil, err } - for _, t := range trackers { + for _, t := range resourceTrackers { resources[t.Type+":"+t.ID] = t } } for k, t := range resources { - if t.done { + if t.Done { delete(resources, k) } } @@ -178,32 +179,32 @@ func (d *clusterDiscoveryGCE) findInstanceTemplates() ([]*compute.InstanceTempla return matches, nil } -func (d *clusterDiscoveryGCE) listGCEInstanceTemplates() ([]*ResourceTracker, error) { - var trackers []*ResourceTracker +func (d *clusterDiscoveryGCE) listGCEInstanceTemplates() ([]*tracker.Resource, error) { + var resourceTrackers []*tracker.Resource templates, err := d.findInstanceTemplates() if err != nil { return nil, err } for _, t := range templates { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: t.Name, ID: t.Name, Type: typeInstanceTemplate, - deleter: deleteGCEInstanceTemplate, - obj: t, + Deleter: deleteGCEInstanceTemplate, + Obj: t, } glog.V(4).Infof("Found resource: %s", t.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func deleteGCEInstanceTemplate(cloud fi.Cloud, r *ResourceTracker) error { +func deleteGCEInstanceTemplate(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.InstanceTemplate) + t := r.Obj.(*compute.InstanceTemplate) glog.V(2).Infof("Deleting GCE InstanceTemplate %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -223,11 +224,11 @@ func deleteGCEInstanceTemplate(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listInstanceGroupManagersAndInstances() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listInstanceGroupManagersAndInstances() ([]*tracker.Resource, error) { c := d.gceCloud project := c.Project() - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource instanceTemplates := make(map[string]*compute.InstanceTemplate) { @@ -251,24 +252,24 @@ func (d *clusterDiscoveryGCE) listInstanceGroupManagersAndInstances() ([]*Resour continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: mig.Name, ID: zoneName + "/" + mig.Name, Type: typeInstanceGroupManager, - deleter: deleteInstanceGroupManager, - obj: mig, + Deleter: deleteInstanceGroupManager, + Obj: mig, } - tracker.blocks = append(tracker.blocks, typeInstanceTemplate+":"+instanceTemplate.Name) + resourceTracker.Blocks = append(resourceTracker.Blocks, typeInstanceTemplate+":"+instanceTemplate.Name) glog.V(4).Infof("Found resource: %s", mig.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) instanceTrackers, err := d.listManagedInstances(mig) if err != nil { return fmt.Errorf("error listing instances in InstanceGroupManager: %v", err) } - trackers = append(trackers, instanceTrackers...) + resourceTrackers = append(resourceTrackers, instanceTrackers...) } return nil }) @@ -278,12 +279,12 @@ func (d *clusterDiscoveryGCE) listInstanceGroupManagersAndInstances() ([]*Resour } } - return trackers, nil + return resourceTrackers, nil } -func deleteInstanceGroupManager(cloud fi.Cloud, r *ResourceTracker) error { +func deleteInstanceGroupManager(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.InstanceGroupManager) + t := r.Obj.(*compute.InstanceGroupManager) glog.V(2).Infof("Deleting GCE InstanceGroupManager %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -305,11 +306,11 @@ func deleteInstanceGroupManager(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listManagedInstances(igm *compute.InstanceGroupManager) ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listManagedInstances(igm *compute.InstanceGroupManager) ([]*tracker.Resource, error) { c := d.gceCloud project := c.Project() - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource zoneName := gce.LastComponent(igm.Zone) @@ -322,20 +323,20 @@ func (d *clusterDiscoveryGCE) listManagedInstances(igm *compute.InstanceGroupMan for _, i := range instances.ManagedInstances { name := gce.LastComponent(i.Instance) - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: name, ID: zoneName + "/" + name, Type: typeInstance, - deleter: deleteManagedInstance, - obj: i.Instance, + Deleter: deleteManagedInstance, + Obj: i.Instance, } // We don't block deletion of the instance group manager - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } // findGCEDisks finds all Disks that are associated with the current cluster @@ -383,36 +384,36 @@ func (d *clusterDiscoveryGCE) findGCEDisks() ([]*compute.Disk, error) { return matches, nil } -func (d *clusterDiscoveryGCE) listGCEDisks() ([]*ResourceTracker, error) { - var trackers []*ResourceTracker +func (d *clusterDiscoveryGCE) listGCEDisks() ([]*tracker.Resource, error) { + var resourceTrackers []*tracker.Resource disks, err := d.findGCEDisks() if err != nil { return nil, err } for _, t := range disks { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: t.Name, ID: t.Name, Type: typeDisk, - deleter: deleteGCEDisk, - obj: t, + Deleter: deleteGCEDisk, + Obj: t, } for _, u := range t.Users { - tracker.blocked = append(tracker.blocked, typeInstance+":"+gce.LastComponent(t.Zone)+"/"+gce.LastComponent(u)) + resourceTracker.Blocked = append(resourceTracker.Blocked, typeInstance+":"+gce.LastComponent(t.Zone)+"/"+gce.LastComponent(u)) } glog.V(4).Infof("Found resource: %s", t.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } - return trackers, nil + return resourceTrackers, nil } -func deleteGCEDisk(cloud fi.Cloud, r *ResourceTracker) error { +func deleteGCEDisk(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.Disk) + t := r.Obj.(*compute.Disk) glog.V(2).Infof("Deleting GCE Disk %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -432,10 +433,10 @@ func deleteGCEDisk(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listTargetPools() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listTargetPools() ([]*tracker.Resource, error) { c := d.gceCloud - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource ctx := context.Background() @@ -445,16 +446,16 @@ func (d *clusterDiscoveryGCE) listTargetPools() ([]*ResourceTracker, error) { continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: tp.Name, ID: tp.Name, Type: typeTargetPool, - deleter: deleteTargetPool, - obj: tp, + Deleter: deleteTargetPool, + Obj: tp, } glog.V(4).Infof("Found resource: %s", tp.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } return nil @@ -463,12 +464,12 @@ func (d *clusterDiscoveryGCE) listTargetPools() ([]*ResourceTracker, error) { return nil, fmt.Errorf("error listing TargetPools: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func deleteTargetPool(cloud fi.Cloud, r *ResourceTracker) error { +func deleteTargetPool(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.TargetPool) + t := r.Obj.(*compute.TargetPool) glog.V(2).Infof("Deleting GCE TargetPool %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -490,10 +491,10 @@ func deleteTargetPool(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listForwardingRules() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listForwardingRules() ([]*tracker.Resource, error) { c := d.gceCloud - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource ctx := context.Background() @@ -503,24 +504,24 @@ func (d *clusterDiscoveryGCE) listForwardingRules() ([]*ResourceTracker, error) continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: fr.Name, ID: fr.Name, Type: typeForwardingRule, - deleter: deleteForwardingRule, - obj: fr, + Deleter: deleteForwardingRule, + Obj: fr, } if fr.Target != "" { - tracker.blocks = append(tracker.blocks, typeTargetPool+":"+gce.LastComponent(fr.Target)) + resourceTracker.Blocks = append(resourceTracker.Blocks, typeTargetPool+":"+gce.LastComponent(fr.Target)) } if fr.IPAddress != "" { - tracker.blocks = append(tracker.blocks, typeAddress+":"+gce.LastComponent(fr.IPAddress)) + resourceTracker.Blocks = append(resourceTracker.Blocks, typeAddress+":"+gce.LastComponent(fr.IPAddress)) } glog.V(4).Infof("Found resource: %s", fr.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } return nil }) @@ -528,12 +529,12 @@ func (d *clusterDiscoveryGCE) listForwardingRules() ([]*ResourceTracker, error) return nil, fmt.Errorf("error listing ForwardingRules: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func deleteForwardingRule(cloud fi.Cloud, r *ResourceTracker) error { +func deleteForwardingRule(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.ForwardingRule) + t := r.Obj.(*compute.ForwardingRule) glog.V(2).Infof("Deleting GCE ForwardingRule %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -553,9 +554,9 @@ func deleteForwardingRule(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func deleteManagedInstance(cloud fi.Cloud, r *ResourceTracker) error { +func deleteManagedInstance(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - selfLink := r.obj.(string) + selfLink := r.Obj.(string) glog.V(2).Infof("Deleting GCE Instance %s", selfLink) u, err := gce.ParseGoogleCloudURL(selfLink) @@ -575,10 +576,10 @@ func deleteManagedInstance(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listRoutes(resources map[string]*ResourceTracker) ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listRoutes(resources map[string]*tracker.Resource) ([]*tracker.Resource, error) { c := d.gceCloud - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource instances := sets.NewString() for _, resource := range resources { @@ -619,21 +620,21 @@ func (d *clusterDiscoveryGCE) listRoutes(resources map[string]*ResourceTracker) } if remove { - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: r.Name, ID: r.Name, Type: typeRoute, - deleter: deleteRoute, - obj: r, + Deleter: deleteRoute, + Obj: r, } // We don't need to block //if r.NextHopInstance != "" { - // tracker.blocked = append(tracker.blocks, typeInstance+":"+gce.LastComponent(r.NextHopInstance)) + // resourceTracker.Blocked = append(resourceTracker.Blocks, typeInstance+":"+gce.LastComponent(r.NextHopInstance)) //} glog.V(4).Infof("Found resource: %s", r.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } } @@ -642,12 +643,12 @@ func (d *clusterDiscoveryGCE) listRoutes(resources map[string]*ResourceTracker) if err != nil { return nil, fmt.Errorf("error listing Routes: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func deleteRoute(cloud fi.Cloud, r *ResourceTracker) error { +func deleteRoute(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.Route) + t := r.Obj.(*compute.Route) glog.V(2).Infof("Deleting GCE Route %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -667,10 +668,10 @@ func deleteRoute(cloud fi.Cloud, r *ResourceTracker) error { return c.WaitForOp(op) } -func (d *clusterDiscoveryGCE) listAddresses() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listAddresses() ([]*tracker.Resource, error) { c := d.gceCloud - var trackers []*ResourceTracker + var resourceTrackers []*tracker.Resource ctx := context.Background() @@ -681,16 +682,16 @@ func (d *clusterDiscoveryGCE) listAddresses() ([]*ResourceTracker, error) { continue } - tracker := &ResourceTracker{ + resourceTracker := &tracker.Resource{ Name: a.Name, ID: a.Name, Type: typeAddress, - deleter: deleteAddress, - obj: a, + Deleter: deleteAddress, + Obj: a, } glog.V(4).Infof("Found resource: %s", a.SelfLink) - trackers = append(trackers, tracker) + resourceTrackers = append(resourceTrackers, resourceTracker) } return nil }) @@ -698,12 +699,12 @@ func (d *clusterDiscoveryGCE) listAddresses() ([]*ResourceTracker, error) { return nil, fmt.Errorf("error listing Addresses: %v", err) } - return trackers, nil + return resourceTrackers, nil } -func deleteAddress(cloud fi.Cloud, r *ResourceTracker) error { +func deleteAddress(cloud fi.Cloud, r *tracker.Resource) error { c := cloud.(gce.GCECloud) - t := r.obj.(*compute.Address) + t := r.Obj.(*compute.Address) glog.V(2).Infof("Deleting GCE Address %s", t.SelfLink) u, err := gce.ParseGoogleCloudURL(t.SelfLink) @@ -733,7 +734,7 @@ func (d *clusterDiscoveryGCE) matchesClusterName(name string) bool { return name == gce.SafeObjectName(id, d.clusterName) } -func (d *clusterDiscoveryGCE) listGCEDNSZone() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryGCE) listGCEDNSZone() ([]*tracker.Resource, error) { if dns.IsGossipHostname(d.clusterName) { return nil, nil } @@ -743,13 +744,13 @@ func (d *clusterDiscoveryGCE) listGCEDNSZone() ([]*ResourceTracker, error) { return nil, err } - return []*ResourceTracker{ + return []*tracker.Resource{ { Name: zone.Name(), ID: zone.Name(), Type: "DNS Zone", - deleter: d.deleteDNSZone, - obj: zone, + Deleter: d.deleteDNSZone, + Obj: zone, }, }, nil } @@ -779,8 +780,8 @@ func (d *clusterDiscoveryGCE) findDNSZone() (dnsprovider.Zone, error) { return nil, fmt.Errorf("DNS Zone for cluster %s could not be found", d.clusterName) } -func (d *clusterDiscoveryGCE) deleteDNSZone(cloud fi.Cloud, r *ResourceTracker) error { - clusterZone := r.obj.(dnsprovider.Zone) +func (d *clusterDiscoveryGCE) deleteDNSZone(cloud fi.Cloud, r *tracker.Resource) error { + clusterZone := r.Obj.(dnsprovider.Zone) rrs, supported := clusterZone.ResourceRecordSets() if !supported { diff --git a/pkg/resources/tracker/tracker.go b/pkg/resources/tracker/tracker.go new file mode 100644 index 0000000000000..fda7145d65b4f --- /dev/null +++ b/pkg/resources/tracker/tracker.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tracker + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +type Resource struct { + Name string + Type string + ID string + + // If true, this resource is not owned by the cluster + Shared bool + + Blocks []string + Blocked []string + Done bool + + Deleter func(cloud fi.Cloud, tracker *Resource) error + GroupKey string + GroupDeleter func(cloud fi.Cloud, trackers []*Resource) error + + Dumper func(r *Resource) (interface{}, error) + + Obj interface{} +} diff --git a/pkg/resources/vsphere.go b/pkg/resources/vsphere.go index 6a5bd075eaabe..3a1fa9ed354a4 100644 --- a/pkg/resources/vsphere.go +++ b/pkg/resources/vsphere.go @@ -21,6 +21,7 @@ import ( "github.com/golang/glog" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" + "k8s.io/kops/pkg/resources/tracker" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/vsphere" ) @@ -35,12 +36,12 @@ type clusterDiscoveryVSphere struct { clusterName string } -type vsphereListFn func() ([]*ResourceTracker, error) +type vsphereListFn func() ([]*tracker.Resource, error) -func (c *ClusterResources) listResourcesVSphere() (map[string]*ResourceTracker, error) { +func (c *ClusterResources) listResourcesVSphere() (map[string]*tracker.Resource, error) { vsphereCloud := c.Cloud.(*vsphere.VSphereCloud) - resources := make(map[string]*ResourceTracker) + resources := make(map[string]*tracker.Resource) d := &clusterDiscoveryVSphere{ cloud: c.Cloud, @@ -65,7 +66,7 @@ func (c *ClusterResources) listResourcesVSphere() (map[string]*ResourceTracker, return resources, nil } -func (d *clusterDiscoveryVSphere) listVMs() ([]*ResourceTracker, error) { +func (d *clusterDiscoveryVSphere) listVMs() ([]*tracker.Resource, error) { c := d.vsphereCloud regexForMasterVMs := "*" + "." + "masters" + "." + d.clusterName + "*" @@ -79,25 +80,25 @@ func (d *clusterDiscoveryVSphere) listVMs() ([]*ResourceTracker, error) { glog.Warning(err) } - var trackers []*ResourceTracker + var trackers []*tracker.Resource for _, vm := range vms { - tracker := &ResourceTracker{ + tracker := &tracker.Resource{ Name: vm.Name(), ID: vm.Name(), Type: typeVM, - deleter: deleteVM, + Deleter: deleteVM, Dumper: DumpVMInfo, - obj: vm, + Obj: vm, } trackers = append(trackers, tracker) } return trackers, nil } -func deleteVM(cloud fi.Cloud, r *ResourceTracker) error { +func deleteVM(cloud fi.Cloud, r *tracker.Resource) error { vsphereCloud := cloud.(*vsphere.VSphereCloud) - vm := r.obj.(*object.VirtualMachine) + vm := r.Obj.(*object.VirtualMachine) task, err := vm.PowerOff(context.TODO()) if err != nil { @@ -120,14 +121,14 @@ func deleteVM(cloud fi.Cloud, r *ResourceTracker) error { return nil } -func DumpVMInfo(r *ResourceTracker) (interface{}, error) { +func DumpVMInfo(r *tracker.Resource) (interface{}, error) { data := make(map[string]interface{}) data["id"] = r.ID data["type"] = r.Type - data["raw"] = r.obj + data["raw"] = r.Obj return data, nil } -func GetResourceTrackerKey(t *ResourceTracker) string { +func GetResourceTrackerKey(t *tracker.Resource) string { return t.Type + ":" + t.ID }