Skip to content

Commit

Permalink
azure: use finalizers for resource cleanup (#285)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kdomanski
kdomanski committed Jul 25, 2018
1 parent e2eacc0 commit c73d146
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 36 deletions.
52 changes: 21 additions & 31 deletions pkg/cloudprovider/provider/azure/create_delete_resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID typ
}

for _, iface := range allInterfaces {
if iface.Tags != nil && *iface.Tags[machineUIDTag] == string(machineUID) {
if iface.Tags != nil && iface.Tags[machineUIDTag] != nil && *iface.Tags[machineUIDTag] == string(machineUID) {
future, err := ifClient.Delete(ctx, c.ResourceGroup, *iface.Name)
if err != nil {
return err
Expand Down Expand Up @@ -104,7 +104,7 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID)
}

for _, vm := range allServers {
if vm.Tags != nil && *vm.Tags[machineUIDTag] == string(machineUID) {
if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(machineUID) {
future, err := vmClient.Delete(ctx, c.ResourceGroup, *vm.Name)
if err != nil {
return err
Expand Down Expand Up @@ -138,7 +138,7 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI
}

for _, disk := range allDisks {
if disk.Tags != nil && *disk.Tags[machineUIDTag] == string(machineUID) {
if disk.Tags != nil && disk.Tags[machineUIDTag] != nil && *disk.Tags[machineUIDTag] == string(machineUID) {
future, err := disksClient.Delete(ctx, c.ResourceGroup, *disk.Name)
if err != nil {
return err
Expand All @@ -153,10 +153,11 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI
return nil
}

func createPublicIPAddress(ctx context.Context, ipName string, machineUID types.UID, c *config) (network.PublicIPAddress, error) {
func createPublicIPAddress(ctx context.Context, ipName string, machineUID types.UID, c *config) (*network.PublicIPAddress, error) {
glog.Infof("Creating public IP %q", ipName)
ipClient, err := getIPClient(c)
if err != nil {
return network.PublicIPAddress{}, err
return nil, err
}

ipParams := network.PublicIPAddress{
Expand All @@ -170,24 +171,29 @@ func createPublicIPAddress(ctx context.Context, ipName string, machineUID types.
}
future, err := ipClient.CreateOrUpdate(ctx, c.ResourceGroup, ipName, ipParams)
if err != nil {
return network.PublicIPAddress{}, fmt.Errorf("failed to create public IP address: %v", err)
return nil, fmt.Errorf("failed to create public IP address: %v", err)
}

err = future.WaitForCompletion(ctx, ipClient.Client)
if err != nil {
return network.PublicIPAddress{}, fmt.Errorf("failed to retrieve public IP address creation result: %v", err)
return nil, fmt.Errorf("failed to retrieve public IP address creation result: %v", err)
}

return future.Result(*ipClient)
}
if _, err = future.Result(*ipClient); err != nil {
return nil, fmt.Errorf("failed to create public IP address: %v", err)
}

func getPublicIPAddress(ctx context.Context, ipName string, c *config) (*network.PublicIPAddress, error) {
ipClient, err := getIPClient(c)
glog.Infof("Fetching info for IP address %q", ipName)
ip, err := getPublicIPAddress(ctx, ipName, c.ResourceGroup, ipClient)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to fetch info about public IP %q: %v", ipName, err)
}

ip, err := ipClient.Get(ctx, c.ResourceGroup, ipName, "")
return ip, nil
}

func getPublicIPAddress(ctx context.Context, ipName string, resourceGroup string, ipClient *network.PublicIPAddressesClient) (*network.PublicIPAddress, error) {
ip, err := ipClient.Get(ctx, resourceGroup, ipName, "")
if err != nil {
return nil, err
}
Expand All @@ -204,7 +210,7 @@ func getSubnet(ctx context.Context, c *config) (network.Subnet, error) {
return subnetsClient.Get(ctx, c.ResourceGroup, c.VNetName, c.SubnetName, "")
}

func createNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config) (network.Interface, error) {
func createNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP *network.PublicIPAddress) (network.Interface, error) {
ifClient, err := getInterfacesClient(config)
if err != nil {
return network.Interface{}, fmt.Errorf("failed to create interfaces client: %v", err)
Expand All @@ -215,22 +221,6 @@ func createNetworkInterface(ctx context.Context, ifName string, machineUID types
return network.Interface{}, fmt.Errorf("failed to fetch subnet: %v", err)
}

var ip *network.PublicIPAddress
if config.AssignPublicIP {
glog.Infof("Creating public IP for interface %q", ifName)
ipName := ifName + "-pubip"
_, err = createPublicIPAddress(ctx, ipName, machineUID, config)
if err != nil {
return network.Interface{}, fmt.Errorf("failed to create public IP: %v", err)
}

glog.Infof("Fetching info for IP address %q", ipName)
ip, err = getPublicIPAddress(ctx, ipName, config)
if err != nil {
return network.Interface{}, fmt.Errorf("failed to fetch info about public IP %q: %v", ipName, err)
}
}

ifSpec := network.Interface{
Name: to.StringPtr(ifName),
Location: &config.Location,
Expand All @@ -241,7 +231,7 @@ func createNetworkInterface(ctx context.Context, ifName string, machineUID types
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
Subnet: &subnet,
PrivateIPAllocationMethod: network.Dynamic,
PublicIPAddress: ip,
PublicIPAddress: publicIP,
},
},
},
Expand Down
72 changes: 67 additions & 5 deletions pkg/cloudprovider/provider/azure/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ import (
"strings"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-04-01/network"
"github.com/Azure/go-autorest/autorest/to"
"github.com/golang/glog"

"github.com/kubermatic/machine-controller/pkg/cloudprovider/cloud"
"github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh"
cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors"
"github.com/kubermatic/machine-controller/pkg/cloudprovider/instance"
kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes"
"github.com/kubermatic/machine-controller/pkg/machines/v1alpha1"
"github.com/kubermatic/machine-controller/pkg/providerconfig"

Expand All @@ -26,6 +28,11 @@ import (
const (
machineUIDTag = "Machine-UID"
adminUserName = "kubermatic"

finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip"
finalizerNIC = "kubermatic.io/cleanup-azure-nic"
finalizerDisks = "kubermatic.io/cleanup-azure-disks"
finalizerVM = "kubermatic.io/cleanup-azure-vm"
)

type provider struct {
Expand Down Expand Up @@ -301,7 +308,7 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec,
return spec, false, nil
}

func (p *provider) Create(machine *v1alpha1.Machine, _ cloud.MachineUpdater, userdata string) (instance.Instance, error) {
func (p *provider) Create(machine *v1alpha1.Machine, update cloud.MachineUpdater, userdata string) (instance.Instance, error) {
config, providerCfg, err := p.getConfig(machine.Spec.ProviderConfig)
if err != nil {
return nil, cloudprovidererrors.TerminalError{
Expand All @@ -326,8 +333,31 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ cloud.MachineUpdater, use
return nil, fmt.Errorf("failed to generate ssh key: %v", err)
}

if !kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) {
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerPublicIP)
}); err != nil {
return nil, err
}
}
ifaceName := machine.Spec.Name + "-netiface"
iface, err := createNetworkInterface(context.TODO(), ifaceName, machine.UID, config)
publicIPName := ifaceName + "-pubip"
var publicIP *network.PublicIPAddress
if config.AssignPublicIP {
publicIP, err = createPublicIPAddress(context.TODO(), publicIPName, machine.UID, config)
if err != nil {
return nil, fmt.Errorf("failed to create public IP: %v", err)
}
}

if !kuberneteshelper.HasFinalizer(machine, finalizerNIC) {
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerNIC)
}); err != nil {
return nil, err
}
}
iface, err := createNetworkInterface(context.TODO(), ifaceName, machine.UID, config, publicIP)
if err != nil {
return nil, fmt.Errorf("failed to generate main network interface: %v", err)
}
Expand Down Expand Up @@ -372,6 +402,20 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ cloud.MachineUpdater, use
}

glog.Infof("Creating machine %q", machine.Spec.Name)
if !kuberneteshelper.HasFinalizer(machine, finalizerDisks) {
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerDisks)
}); err != nil {
return nil, err
}
}
if !kuberneteshelper.HasFinalizer(machine, finalizerVM) {
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerVM)
}); err != nil {
return nil, err
}
}
future, err := vmClient.CreateOrUpdate(context.TODO(), config.ResourceGroup, machine.Spec.Name, vmSpec)
if err != nil {
return nil, fmt.Errorf("trying to create a VM: %v", err)
Expand Down Expand Up @@ -406,33 +450,51 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ cloud.MachineUpdater, use
return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil
}

func (p *provider) Delete(machine *v1alpha1.Machine, _ cloud.MachineUpdater, instance instance.Instance) error {
func (p *provider) Delete(machine *v1alpha1.Machine, update cloud.MachineUpdater, instance instance.Instance) error {
config, _, err := p.getConfig(machine.Spec.ProviderConfig)
if err != nil {
return fmt.Errorf("failed to parse MachineSpec: %v", err)
}

// TODO: Disassociate and remove the resources in reverse order, to prevent
// orphaned NICs, IPs and disks in case the operation is interrupted in the middle.
glog.Infof("deleting VM %q", machine.Name)
if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil {
return fmt.Errorf("Is failed to remove public IP addresses of machine %q: %v", machine.Name, err)
}
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM)
}); err != nil {
return err
}

glog.Infof("deleting disks of VM %q", machine.Name)
if err = deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil {
return fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err)
}
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks)
}); err != nil {
return err
}

glog.Infof("deleting network interfaces of VM %q", machine.Name)
if err = deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil {
return fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err)
}
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC)
}); err != nil {
return err
}

glog.Infof("deleting public IP addresses of VM %q", machine.Name)
if err = deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil {
return fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err)
}
if machine, err = update(machine.Name, func(updatedMachine *v1alpha1.Machine) {
updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP)
}); err != nil {
return err
}

return nil
}
Expand Down
18 changes: 18 additions & 0 deletions pkg/kubernetes/helper.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package kubernetes

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
)

// HasFinalizer tells if a object has the given finalizer
func HasFinalizer(o metav1.Object, name string) bool {
return sets.NewString(o.GetFinalizers()...).Has(name)
}

// RemoveFinalizer removes the given finalizer and returns the cleaned list
func RemoveFinalizer(finalizers []string, toRemove string) []string {
set := sets.NewString(finalizers...)
set.Delete(toRemove)
return set.List()
}

0 comments on commit c73d146

Please sign in to comment.