Skip to content

Commit

Permalink
fix(native): fix delete native provider's cluster
Browse files Browse the repository at this point in the history 
    1. check cluster exist
    2. sync .state when finish delete the cluster

Signed-off-by: Jason-ZW <zhenyang@rancher.com>
  • Loading branch information
@rancher-sy-bot
rancher-sy-bot committed Dec 23, 2020
1 parent 61f9233 commit 0b9e36c
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 57 deletions.
35 changes: 15 additions & 20 deletions pkg/cluster/cluster.go
View file
Expand Up @@ -418,19 +418,16 @@ func DeleteState(name string, provider string) error {
return utils.WriteYaml(r, v, common.StateFile)
}

func UninstallK3sCluster(cluster *types.Cluster) (debugMsg []string, warnMsg []string, err error) {
func UninstallK3sCluster(cluster *types.Cluster) (warnMsg []string, err error) {
for _, workerNode := range cluster.WorkerNodes {
m, e := execute(&hosts.Host{Node: workerNode}, []string{workerUninstallCommand})
if e == nil {
debugMsg = append(debugMsg, fmt.Sprintf("worker node %s: %s", workerNode.InstanceID, m))
} else {
_, e := execute(&hosts.Host{Node: workerNode}, []string{workerUninstallCommand})
if e != nil {
warnMsg = append(warnMsg, fmt.Sprintf("failed to uninstall k3s on worker node %s: %s", workerNode.InstanceID, e.Error()))
}
}
for _, masterNode := range cluster.MasterNodes {
if m, e := execute(&hosts.Host{Node: masterNode}, []string{masterUninstallCommand}); e == nil {
debugMsg = append(debugMsg, fmt.Sprintf("master node %s: %s", masterNode.InstanceID, m))
} else {
_, e := execute(&hosts.Host{Node: masterNode}, []string{masterUninstallCommand})
if e != nil {
warnMsg = append(warnMsg, fmt.Sprintf("failed to uninstall k3s on master node %s: %s", masterNode.InstanceID, e.Error()))
}
}
Expand All @@ -439,18 +436,16 @@ func UninstallK3sCluster(cluster *types.Cluster) (debugMsg []string, warnMsg []s
return
}

func UninstallK3sNodes(nodes []types.Node) (debugMsg []string, warnMsg []string) {
func UninstallK3sNodes(nodes []types.Node) (warnMsg []string) {
for _, node := range nodes {
if node.Master {
if m, e := execute(&hosts.Host{Node: node}, []string{masterUninstallCommand}); e == nil {
debugMsg = append(debugMsg, fmt.Sprintf("master node %s: %s", node.InstanceID, m))
} else {
_, e := execute(&hosts.Host{Node: node}, []string{masterUninstallCommand})
if e != nil {
warnMsg = append(warnMsg, fmt.Sprintf("failed to uninstall k3s on master node %s: %s", node.InstanceID, e.Error()))
}
} else {
if m, e := execute(&hosts.Host{Node: node}, []string{workerUninstallCommand}); e == nil {
debugMsg = append(debugMsg, fmt.Sprintf("worker node %s: %s", node.InstanceID, m))
} else {
_, e := execute(&hosts.Host{Node: node}, []string{workerUninstallCommand})
if e != nil {
warnMsg = append(warnMsg, fmt.Sprintf("failed to uninstall k3s on worker node %s: %s", node.InstanceID, e.Error()))
}
}
Expand Down Expand Up @@ -565,7 +560,7 @@ func initMaster(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, cluste
}
}

logger.Debugf("[cluster] k3s master command: %s", fmt.Sprintf(initCommand, k3sScript, k3sMirror, cluster.Token,
logger.Debugf("[cluster] k3s master command: %s\n", fmt.Sprintf(initCommand, k3sScript, k3sMirror, cluster.Token,
"--cluster-init", ip, strings.TrimSpace(extraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel)))

if _, err := execute(&hosts.Host{Node: master}, []string{fmt.Sprintf(initCommand, k3sScript, k3sMirror,
Expand Down Expand Up @@ -597,7 +592,7 @@ func initAdditionalMaster(k3sScript, k3sMirror, dockerMirror, ip, extraArgs stri

sortedExtraArgs += " " + extraArgs

logger.Debugf("[cluster] k3s additional master command: %s", fmt.Sprintf(joinCommand, k3sScript, k3sMirror,
logger.Debugf("[cluster] k3s additional master command: %s\n", fmt.Sprintf(joinCommand, k3sScript, k3sMirror,
ip, cluster.Token, strings.TrimSpace(sortedExtraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel)))

if _, err := execute(&hosts.Host{Node: master}, []string{fmt.Sprintf(joinCommand, k3sScript, k3sMirror, ip,
Expand Down Expand Up @@ -626,7 +621,7 @@ func initWorker(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, do

sortedExtraArgs += " " + extraArgs

logger.Debugf("[cluster] k3s worker command: %s", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, cluster.IP,
logger.Debugf("[cluster] k3s worker command: %s\n", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, cluster.IP,
cluster.Token, strings.TrimSpace(sortedExtraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel)))

if _, err := execute(&hosts.Host{Node: worker}, []string{fmt.Sprintf(joinCommand, k3sScript, k3sMirror, cluster.IP,
Expand Down Expand Up @@ -667,7 +662,7 @@ func joinMaster(k3sScript, k3sMirror, dockerMirror,

sortedExtraArgs += " " + extraArgs

logger.Debugf("[cluster] k3s master command: %s", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.IP,
logger.Debugf("[cluster] k3s master command: %s\n", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.IP,
merged.Token, strings.TrimSpace(sortedExtraArgs), genK3sVersion(merged.K3sVersion, merged.K3sChannel)))

// for now, use the workerCommand to join the additional master server node.
Expand Down Expand Up @@ -697,7 +692,7 @@ func joinWorker(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, do

sortedExtraArgs += " " + extraArgs

logger.Debugf("[cluster] k3s worker command: %s", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.IP,
logger.Debugf("[cluster] k3s worker command: %s\n", fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.IP,
merged.Token, strings.TrimSpace(sortedExtraArgs), genK3sVersion(merged.K3sVersion, merged.K3sChannel)))

if _, err := execute(&hosts.Host{Node: full}, []string{fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.IP,
Expand Down
24 changes: 12 additions & 12 deletions pkg/providers/alibaba/alibaba.go
View file
Expand Up @@ -531,7 +531,7 @@ func (p *Alibaba) runInstances(num int, master bool, password string) error {
func (p *Alibaba) deleteCluster(f bool) error {
exist, ids, err := p.IsClusterExist()
if err != nil && !f {
return fmt.Errorf("[%s] calling deleteInstance error, msg: %v", p.GetProviderName(), err)
return fmt.Errorf("[%s] calling deleteCluster error, msg: %v", p.GetProviderName(), err)
}
if !exist {
return fmt.Errorf("[%s] calling preflight error: cluster name `%s` do not exist", p.GetProviderName(), p.Name)
Expand Down Expand Up @@ -673,7 +673,7 @@ func (p *Alibaba) assembleInstanceStatus(ssh *types.SSH, uploadKeyPair bool, pub
v.SSH = *ssh
// check upload keypair
if uploadKeyPair {
p.logger.Debugf("[%s] Waiting for upload keypair...", p.GetProviderName())
p.logger.Debugf("[%s] Waiting for upload keypair...\n", p.GetProviderName())
if err := p.uploadKeyPair(v, publicKey); err != nil {
return nil, err
}
Expand Down Expand Up @@ -1328,7 +1328,7 @@ func (p *Alibaba) createKeyPair(ssh *types.SSH) (string, error) {
}

func (p *Alibaba) generateDefaultVPC() error {
p.logger.Debugf("[%s] generate default vpc %s in region %s", p.GetProviderName(), vpcName, p.Region)
p.logger.Debugf("[%s] generate default vpc %s in region %s\n", p.GetProviderName(), vpcName, p.Region)
request := vpc.CreateCreateVpcRequest()
request.Scheme = "https"
request.RegionId = p.Region
Expand Down Expand Up @@ -1359,7 +1359,7 @@ func (p *Alibaba) generateDefaultVPC() error {
return fmt.Errorf("[%s] error tag default vpc %s, got error: %v", p.GetProviderName(), response.VpcId, err)
}

p.logger.Debugf("[%s] waiting for vpc %s available", p.GetProviderName(), p.Vpc)
p.logger.Debugf("[%s] waiting for vpc %s available\n", p.GetProviderName(), p.Vpc)
// wait for vpc available
err = utils.WaitFor(p.isVPCAvailable)

Expand Down Expand Up @@ -1397,7 +1397,7 @@ func (p *Alibaba) isVSwitchAvailable() (bool, error) {
}

func (p *Alibaba) generateDefaultVSwitch() error {
p.logger.Debugf("[%s] generate default vswitch %s for vpc %s in region %s, zone %s", p.GetProviderName(), vSwitchName, vpcName, p.Region, p.Zone)
p.logger.Debugf("[%s] generate default vswitch %s for vpc %s in region %s, zone %s\n", p.GetProviderName(), vSwitchName, vpcName, p.Region, p.Zone)
request := vpc.CreateCreateVSwitchRequest()
request.Scheme = "https"

Expand Down Expand Up @@ -1431,7 +1431,7 @@ func (p *Alibaba) generateDefaultVSwitch() error {
return fmt.Errorf("[%s] error tag default vswitch %s, got error: %v", p.GetProviderName(), p.VSwitch, err)
}

p.logger.Debugf("[%s] waiting for vswitch %s available", p.GetProviderName(), p.VSwitch)
p.logger.Debugf("[%s] waiting for vswitch %s available\n", p.GetProviderName(), p.VSwitch)
// wait for vswitch available
err = utils.WaitFor(p.isVSwitchAvailable)

Expand Down Expand Up @@ -1501,7 +1501,7 @@ func (p *Alibaba) configNetwork() error {
}

func (p *Alibaba) configSecurityGroup() error {
p.logger.Debugf("[%s] config default security group for %s in region %s", p.GetProviderName(), p.Vpc, p.Region)
p.logger.Debugf("[%s] config default security group for %s in region %s\n", p.GetProviderName(), p.Vpc, p.Region)

if p.Vpc == "" {
// if user didn't set security group, get vpc from vswitch to config default security group
Expand Down Expand Up @@ -1535,7 +1535,7 @@ func (p *Alibaba) configSecurityGroup() error {

if securityGroup == nil {
// create default security group
p.logger.Debugf("[%s] create default security group %s for %s in region %s", p.GetProviderName(), defaultSecurityGroupName, p.Vpc, p.Region)
p.logger.Debugf("[%s] create default security group %s for %s in region %s\n", p.GetProviderName(), defaultSecurityGroupName, p.Vpc, p.Region)
req := ecs.CreateCreateSecurityGroupRequest()
req.Scheme = "https"
req.RegionId = p.Region
Expand All @@ -1553,7 +1553,7 @@ func (p *Alibaba) configSecurityGroup() error {
return fmt.Errorf("[%s] create default security group %s for %s in region %s error: %v", p.GetProviderName(), defaultSecurityGroupName, p.Vpc, p.Region, err)
}
securityGroupID := resp.SecurityGroupId
p.logger.Debugf("[%s] waiting for security group %s available", p.GetProviderName(), securityGroupID)
p.logger.Debugf("[%s] waiting for security group %s available\n", p.GetProviderName(), securityGroupID)
err = utils.WaitFor(func() (bool, error) {
s, err := p.getSecurityGroup(securityGroupID)
if s != nil && err == nil {
Expand Down Expand Up @@ -1598,7 +1598,7 @@ func (p *Alibaba) configDefaultSecurityPermissions(sg *ecs.DescribeSecurityGroup
for _, perm := range sg.Permissions.Permission {
portRange := strings.Split(perm.PortRange, "/")

p.logger.Debugf("[%s] get portRange %v for security group %s", p.GetProviderName(), portRange, sg.SecurityGroupId)
p.logger.Debugf("[%s] get portRange %v for security group %s\n", p.GetProviderName(), portRange, sg.SecurityGroupId)
fromPort, _ := strconv.Atoi(portRange[0])
switch fromPort {
case 22:
Expand Down Expand Up @@ -1684,14 +1684,14 @@ func (p *Alibaba) uploadKeyPair(node types.Node, publicKey string) error {
)
command := fmt.Sprintf("mkdir -p ~/.ssh; echo '%s' > ~/.ssh/authorized_keys", strings.Trim(publicKey, "\n"))

p.logger.Debugf("[%s] upload the public key with command: %s", p.GetProviderName(), command)
p.logger.Debugf("[%s] upload the public key with command: %s\n", p.GetProviderName(), command)

tunnel.Cmd(command)

if err := tunnel.SetStdio(&stdout, &stderr).Run(); err != nil || stderr.String() != "" {
return fmt.Errorf("%w: %s", err, stderr.String())
}
p.logger.Debugf("[%s] upload keypair with output: %s", p.GetProviderName(), stdout.String())
p.logger.Debugf("[%s] upload keypair with output: %s\n", p.GetProviderName(), stdout.String())
return nil
}

Expand Down
24 changes: 13 additions & 11 deletions pkg/providers/native/native.go
View file
Expand Up @@ -271,12 +271,9 @@ func (p *Native) Rollback() error {
p.logger.Debugf("[%s] nodes %s will be rollback\n", p.GetProviderName(), ids)

if len(ids) > 0 {
debugMsg, warnMsg := cluster.UninstallK3sNodes(nodes)
for _, d := range debugMsg {
p.logger.Debugf("[%s] %s", p.GetProviderName(), d)
}
warnMsg := cluster.UninstallK3sNodes(nodes)
for _, w := range warnMsg {
p.logger.Warnf("[%s] %s", p.GetProviderName(), w)
p.logger.Warnf("[%s] %s\n", p.GetProviderName(), w)
}
}

Expand All @@ -288,6 +285,14 @@ func (p *Native) Rollback() error {
func (p *Native) DeleteK3sCluster(f bool) error {
isConfirmed := true

exist, _, err := p.IsClusterExist()
if err != nil && !f {
return fmt.Errorf("[%s] calling deleteCluster error, msg: %v", p.GetProviderName(), err)
}
if !exist {
return fmt.Errorf("[%s] calling preflight error: cluster name `%s` do not exist", p.GetProviderName(), p.Name)
}

if !f {
isConfirmed = utils.AskForConfirmation(fmt.Sprintf("[%s] are you sure to delete cluster %s", p.GetProviderName(), p.Name))
}
Expand All @@ -296,18 +301,15 @@ func (p *Native) DeleteK3sCluster(f bool) error {
p.logger = common.NewLogger(common.Debug)
p.logger.Infof("[%s] executing delete cluster logic...\n", p.GetProviderName())

debugMsg, warnMsg, uninstallErr := cluster.UninstallK3sCluster(&types.Cluster{
warnMsg, uninstallErr := cluster.UninstallK3sCluster(&types.Cluster{
Metadata: p.Metadata,
Options: p.Options,
Status: p.Status,
})

// show debug and warn messages before return error
for _, d := range debugMsg {
p.logger.Debugf("[%s] %s", p.GetProviderName(), d)
}
// show warn messages before return error
for _, w := range warnMsg {
p.logger.Warnf("[%s] %s", p.GetProviderName(), w)
p.logger.Warnf("[%s] %s\n", p.GetProviderName(), w)
}
if uninstallErr != nil {
return uninstallErr
Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/provider.go
View file
Expand Up @@ -70,7 +70,7 @@ func RegisterProvider(name string, p Factory) {
providersMutex.Lock()
defer providersMutex.Unlock()
if _, found := providers[name]; !found {
logrus.Debugf("registered provider %s", name)
logrus.Debugf("registered provider %s\n", name)
providers[name] = p
}
}
Expand Down
26 changes: 13 additions & 13 deletions pkg/providers/tencent/tencent.go
View file
Expand Up @@ -682,7 +682,7 @@ func (p *Tencent) deleteCluster(f bool) error {
}

if err != nil && !f {
return fmt.Errorf("[%s] calling deleteInstance error, msg: %v", p.GetProviderName(), err)
return fmt.Errorf("[%s] calling deleteCluster error, msg: %v", p.GetProviderName(), err)
}

err = cluster.OverwriteCfg(p.Name)
Expand Down Expand Up @@ -860,7 +860,7 @@ func (p *Tencent) assembleInstanceStatus(ssh *types.SSH, uploadKeyPair bool, pub
v.SSH = *ssh
// check upload keypair
if uploadKeyPair {
p.logger.Debugf("[%s] waiting for upload keypair...", p.GetProviderName())
p.logger.Debugf("[%s] waiting for upload keypair...\n", p.GetProviderName())
if err := p.uploadKeyPair(v, publicKey); err != nil {
return nil, err
}
Expand Down Expand Up @@ -1234,7 +1234,7 @@ func (p *Tencent) configNetwork() error {
}

if response != nil && response.Response != nil && len(response.Response.VpcSet) > 0 {
p.logger.Debugf("[%s] find existed default vpc %s for autok3s", p.GetProviderName(), vpcName)
p.logger.Debugf("[%s] find existed default vpc %s for autok3s\n", p.GetProviderName(), vpcName)
defaultVPC := response.Response.VpcSet[0]
p.VpcID = *defaultVPC.VpcId
// find default subnet
Expand All @@ -1257,7 +1257,7 @@ func (p *Tencent) configNetwork() error {
}

if resp != nil && resp.Response != nil && len(resp.Response.SubnetSet) > 0 {
p.logger.Debugf("[%s] find existed default subnet %s for vpc %s", p.GetProviderName(), subnetName, vpcName)
p.logger.Debugf("[%s] find existed default subnet %s for vpc %s\n", p.GetProviderName(), subnetName, vpcName)
p.SubnetID = *resp.Response.SubnetSet[0].SubnetId
} else {
return p.generateDefaultSubnet()
Expand All @@ -1278,7 +1278,7 @@ func (p *Tencent) configNetwork() error {
}

func (p *Tencent) generateDefaultVPC() error {
p.logger.Debugf("[%s] generate default vpc %s in region %s", p.GetProviderName(), vpcName, p.Region)
p.logger.Debugf("[%s] generate default vpc %s in region %s\n", p.GetProviderName(), vpcName, p.Region)
request := vpc.NewCreateVpcRequest()
request.VpcName = tencentCommon.StringPtr(vpcName)
request.CidrBlock = tencentCommon.StringPtr(vpcCidrBlock)
Expand All @@ -1294,13 +1294,13 @@ func (p *Tencent) generateDefaultVPC() error {
}

p.VpcID = *response.Response.Vpc.VpcId
p.logger.Debugf("[%s] generate default vpc %s in region %s successfully", p.GetProviderName(), vpcName, p.Region)
p.logger.Debugf("[%s] generate default vpc %s in region %s successfully\n", p.GetProviderName(), vpcName, p.Region)

return err
}

func (p *Tencent) generateDefaultSubnet() error {
p.logger.Debugf("[%s] generate default subnet %s for vpc %s in region %s", p.GetProviderName(), subnetName, vpcName, p.Region)
p.logger.Debugf("[%s] generate default subnet %s for vpc %s in region %s\n", p.GetProviderName(), subnetName, vpcName, p.Region)
request := vpc.NewCreateSubnetRequest()

request.Tags = []*vpc.Tag{
Expand All @@ -1319,12 +1319,12 @@ func (p *Tencent) generateDefaultSubnet() error {
return fmt.Errorf("[%s] fail to create default subnet for vpc %s in region %s, zone %s: %v", p.GetProviderName(), p.VpcID, p.Region, p.Zone, err)
}
p.SubnetID = *response.Response.Subnet.SubnetId
p.logger.Debugf("[%s] generate default subnet %s for vpc %s in region %s successfully", p.GetProviderName(), subnetName, vpcName, p.Region)
p.logger.Debugf("[%s] generate default subnet %s for vpc %s in region %s successfully\n", p.GetProviderName(), subnetName, vpcName, p.Region)
return nil
}

func (p *Tencent) configSecurityGroup() error {
p.logger.Debugf("[%s] check default security group %s in region %s", p.GetProviderName(), defaultSecurityGroupName, p.Region)
p.logger.Debugf("[%s] check default security group %s in region %s\n", p.GetProviderName(), defaultSecurityGroupName, p.Region)
// find default security group
request := vpc.NewDescribeSecurityGroupsRequest()

Expand All @@ -1351,7 +1351,7 @@ func (p *Tencent) configSecurityGroup() error {

if securityGroupID == "" {
// create default security group
p.logger.Debugf("[%s] create default security group %s in region %s", p.GetProviderName(), defaultSecurityGroupName, p.Region)
p.logger.Debugf("[%s] create default security group %s in region %s\n", p.GetProviderName(), defaultSecurityGroupName, p.Region)
err = p.generateDefaultSecurityGroup()
if err != nil {
return fmt.Errorf("[%s] fail to create default security group %s: %v", p.GetProviderName(), defaultSecurityGroupName, err)
Expand Down Expand Up @@ -1385,7 +1385,7 @@ func (p *Tencent) generateDefaultSecurityGroup() error {
}

func (p *Tencent) configDefaultSecurityPermission() error {
p.logger.Debugf("[%s] check rules of security group %s", p.GetProviderName(), defaultSecurityGroupName)
p.logger.Debugf("[%s] check rules of security group %s\n", p.GetProviderName(), defaultSecurityGroupName)
// get security group rules
request := vpc.NewDescribeSecurityGroupPoliciesRequest()
request.SecurityGroupId = tencentCommon.StringPtr(p.SecurityGroupIds)
Expand Down Expand Up @@ -1613,14 +1613,14 @@ func (p *Tencent) uploadKeyPair(node types.Node, publicKey string) error {
)
command := fmt.Sprintf("mkdir -p ~/.ssh; echo '%s' > ~/.ssh/authorized_keys", strings.Trim(publicKey, "\n"))

p.logger.Debugf("[%s] upload the public key with command: %s", p.GetProviderName(), command)
p.logger.Debugf("[%s] upload the public key with command: %s\n", p.GetProviderName(), command)

tunnel.Cmd(command)

if err := tunnel.SetStdio(&stdout, &stderr).Run(); err != nil || stderr.String() != "" {
return fmt.Errorf("%w: %s", err, stderr.String())
}
p.logger.Debugf("[%s] upload keypair with output: %s", p.GetProviderName(), stdout.String())
p.logger.Debugf("[%s] upload keypair with output: %s\n", p.GetProviderName(), stdout.String())
return nil
}

Expand Down

0 comments on commit 0b9e36c

Please sign in to comment.