From 9b488cec098f1424bbde9b913a53423a0cde62b6 Mon Sep 17 00:00:00 2001 From: Kimmo Lehto Date: Mon, 7 Dec 2020 11:08:49 +0200 Subject: [PATCH 1/3] Upgrade engines on MSR hosts individually --- pkg/product/mke/phase/upgrade_engine.go | 48 +++++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/pkg/product/mke/phase/upgrade_engine.go b/pkg/product/mke/phase/upgrade_engine.go index 9dea0c1c..97fa5d1d 100644 --- a/pkg/product/mke/phase/upgrade_engine.go +++ b/pkg/product/mke/phase/upgrade_engine.go @@ -4,7 +4,9 @@ import ( "fmt" "math" "sync" + "time" + "github.com/Mirantis/mcc/pkg/msr" "github.com/Mirantis/mcc/pkg/phase" "github.com/Mirantis/mcc/pkg/product/mke/api" @@ -51,15 +53,22 @@ func (p *UpgradeEngine) Run() error { // TODO: should we drain? func (p *UpgradeEngine) upgradeEngines() error { var managers api.Hosts - var others api.Hosts + var workers api.Hosts + var msrs api.Hosts for _, h := range p.Hosts { - if h.Role == "manager" { + switch h.Role { + case "manager": managers = append(managers, h) - } else { - others = append(others, h) + case "workers": + managers = append(workers, h) + case "msr": + managers = append(msrs, h) + default: + return fmt.Errorf("%s: unknown role: %s", h, h.Role) } } + // Upgrade managers individually for _, h := range managers { err := p.upgradeEngine(h) if err != nil { @@ -73,15 +82,40 @@ func (p *UpgradeEngine) upgradeEngines() error { } } - // sacrifice 10% of workers for upgrade gods - concurrentUpgrades := int(math.Floor(float64(len(others)) * 0.10)) + // Upgrade MSR hosts individually + for _, h := range msrs { + err := p.upgradeEngine(h) + if err != nil { + return err + } + if h.MSRMetadata.Installed { + err := retry.Do( + func() error { + if _, err := msr.CollectFacts(h); err != nil { + return err + } + return nil + }, + retry.DelayType(retry.CombineDelay(retry.FixedDelay, retry.RandomDelay)), + retry.MaxJitter(time.Second*2), + retry.Delay(time.Second*5), + retry.Attempts(3), + ) + if err != nil { + return err + } + } + } + + // Upgrade worker hosts parallelly in 10% chunks + concurrentUpgrades := int(math.Floor(float64(len(workers)) * 0.10)) if concurrentUpgrades == 0 { concurrentUpgrades = 1 } wp := workerpool.New(concurrentUpgrades) mu := sync.Mutex{} installErrors := &phase.Error{} - for _, w := range others { + for _, w := range workers { h := w wp.Submit(func() { err := p.upgradeEngine(h) From a313462ed4d80b79327f2f20bbbeec7896a21733 Mon Sep 17 00:00:00 2001 From: Kimmo Lehto Date: Mon, 7 Dec 2020 11:20:25 +0200 Subject: [PATCH 2/3] Copy paste fail --- pkg/product/mke/phase/upgrade_engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/product/mke/phase/upgrade_engine.go b/pkg/product/mke/phase/upgrade_engine.go index 97fa5d1d..93b57fae 100644 --- a/pkg/product/mke/phase/upgrade_engine.go +++ b/pkg/product/mke/phase/upgrade_engine.go @@ -60,9 +60,9 @@ func (p *UpgradeEngine) upgradeEngines() error { case "manager": managers = append(managers, h) case "workers": - managers = append(workers, h) + workers = append(workers, h) case "msr": - managers = append(msrs, h) + msrs = append(msrs, h) default: return fmt.Errorf("%s: unknown role: %s", h, h.Role) } From 08898b11af081838a8309142482c44b549cb00ba Mon Sep 17 00:00:00 2001 From: Kimmo Lehto Date: Mon, 7 Dec 2020 11:20:55 +0200 Subject: [PATCH 3/3] Copy paste fail #2 --- pkg/product/mke/phase/upgrade_engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/product/mke/phase/upgrade_engine.go b/pkg/product/mke/phase/upgrade_engine.go index 93b57fae..8c18ab37 100644 --- a/pkg/product/mke/phase/upgrade_engine.go +++ b/pkg/product/mke/phase/upgrade_engine.go @@ -59,7 +59,7 @@ func (p *UpgradeEngine) upgradeEngines() error { switch h.Role { case "manager": managers = append(managers, h) - case "workers": + case "worker": workers = append(workers, h) case "msr": msrs = append(msrs, h)