Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add dedicated phase waiting traffic increase #1252

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions artifacts/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1033,6 +1033,7 @@ spec:
- Initialized
- Waiting
- Progressing
- WaitingTrafficIncrease
- WaitingPromotion
- Promoting
- Finalising
Expand Down
1 change: 1 addition & 0 deletions charts/flagger/crds/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1033,6 +1033,7 @@ spec:
- Initialized
- Waiting
- Progressing
- WaitingTrafficIncrease
- WaitingPromotion
- Promoting
- Finalising
Expand Down
2 changes: 1 addition & 1 deletion docs/gitbook/usage/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ status:
```

The `Promoted` status condition can have one of the following reasons:
Initialized, Waiting, Progressing, WaitingPromotion, Promoting, Finalising, Succeeded or Failed.
Initialized, Waiting, Progressing, WaitingTrafficIncrease, WaitingPromotion, Promoting, Finalising, Succeeded or Failed.
A failed canary will have the promoted status set to `false`,
the reason to `failed` and the last applied spec will be different to the last promoted one.

Expand Down
1 change: 1 addition & 0 deletions kustomize/base/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1033,6 +1033,7 @@ spec:
- Initialized
- Waiting
- Progressing
- WaitingTrafficIncrease
- WaitingPromotion
- Promoting
- Finalising
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/flagger/v1beta1/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ const (
CanaryPhaseWaiting CanaryPhase = "Waiting"
// CanaryPhaseProgressing means the canary analysis is underway
CanaryPhaseProgressing CanaryPhase = "Progressing"
// CanaryPhaseWaitingTrafficIncrease means the canary advancement is paused and is waiting for confirmation to increase traffic to the canary.
CanaryPhaseWaitingTrafficIncrease CanaryPhase = "WaitingTrafficIncrease"
// CanaryPhaseWaitingPromotion means the canary promotion is paused (waiting for confirmation to proceed)
CanaryPhaseWaitingPromotion CanaryPhase = "WaitingPromotion"
// CanaryPhasePromoting means the canary analysis is finished and the primary spec has been updated
Expand Down
3 changes: 3 additions & 0 deletions pkg/canary/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,9 @@ func MakeStatusConditions(cd *flaggerv1.Canary,
case flaggerv1.CanaryPhaseWaiting:
status = corev1.ConditionUnknown
message = "Waiting for approval."
case flaggerv1.CanaryPhaseWaitingTrafficIncrease:
status = corev1.ConditionUnknown
message = "Waiting for approval."
case flaggerv1.CanaryPhaseWaitingPromotion:
status = corev1.ConditionUnknown
message = "Waiting for approval."
Expand Down
12 changes: 9 additions & 3 deletions pkg/controller/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
// check if we should rollback
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
cd.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
cd.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease ||
cd.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion {
if ok := c.runRollbackHooks(cd, cd.Status.Phase); ok {
c.recordEventWarningf(cd, "Rolling back %s.%s manual webhook invoked", cd.Name, cd.Namespace)
Expand Down Expand Up @@ -379,7 +380,9 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}

// check if the number of failed checks reached the threshold
if (cd.Status.Phase == flaggerv1.CanaryPhaseProgressing || cd.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion) &&
if (cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
cd.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease ||
cd.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion) &&
(!retriable || cd.Status.FailedChecks >= cd.GetAnalysisThreshold()) {
if !retriable {
c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v",
Expand All @@ -398,7 +401,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {

// check if the canary success rate is above the threshold
// skip check if no traffic is routed or mirrored to canary
if canaryWeight == 0 && cd.Status.Iterations == 0 &&
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing && canaryWeight == 0 && cd.Status.Iterations == 0 &&
aryan9600 marked this conversation as resolved.
Show resolved Hide resolved
!(cd.GetAnalysis().Mirror && mirrored) {
c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)

Expand Down Expand Up @@ -447,7 +450,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
if c.nextStepWeight(cd, canaryWeight) > 0 {
// run hook only if traffic is not mirrored
if !mirrored {
if promote := c.runConfirmTrafficIncreaseHooks(cd); !promote {
if promote := c.runConfirmTrafficIncreaseHooks(cd, canaryController); !promote {
return
}
}
Expand Down Expand Up @@ -795,6 +798,7 @@ func (c *Controller) shouldAdvance(canary *flaggerv1.Canary, canaryController ca
canary.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
canary.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion ||
canary.Status.Phase == flaggerv1.CanaryPhasePromoting ||
canary.Status.Phase == flaggerv1.CanaryPhaseFinalising {
Expand Down Expand Up @@ -829,6 +833,7 @@ func (c *Controller) shouldAdvance(canary *flaggerv1.Canary, canaryController ca
func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryController canary.Controller, scalerReconciler canary.ScalerReconciler, shouldAdvance bool) bool {
c.recorder.SetStatus(canary, canary.Status.Phase)
if canary.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion ||
canary.Status.Phase == flaggerv1.CanaryPhasePromoting ||
canary.Status.Phase == flaggerv1.CanaryPhaseFinalising {
Expand Down Expand Up @@ -884,6 +889,7 @@ func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryControlle

func (c *Controller) hasCanaryRevisionChanged(canary *flaggerv1.Canary, canaryController canary.Controller) bool {
if canary.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease ||
canary.Status.Phase == flaggerv1.CanaryPhaseWaitingPromotion {
if diff, _ := canaryController.HasTargetChanged(canary); diff {
return true
Expand Down
24 changes: 18 additions & 6 deletions pkg/controller/scheduler_hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,29 @@ import (
"github.com/fluxcd/flagger/pkg/canary"
)

func (c *Controller) runConfirmTrafficIncreaseHooks(canary *flaggerv1.Canary) bool {
func (c *Controller) runConfirmTrafficIncreaseHooks(canary *flaggerv1.Canary, canaryController canary.Controller) bool {
for _, webhook := range canary.GetAnalysis().Webhooks {
if webhook.Type == flaggerv1.ConfirmTrafficIncreaseHook {
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
err := CallWebhook(canary.Name, canary.Namespace, canary.Status.Phase, webhook)
if err != nil {
c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for traffic increase approval %s",
canary.Name, canary.Namespace, webhook.Name)
if !webhook.MuteAlert {
c.alert(canary, "Canary traffic increase is waiting for approval.", false, flaggerv1.SeverityWarn)
if canary.Status.Phase != flaggerv1.CanaryPhaseWaitingTrafficIncrease {
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaitingTrafficIncrease); err != nil {
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
}
c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for traffic increase approval %s",
canary.Name, canary.Namespace, webhook.Name)
if !webhook.MuteAlert {
c.alert(canary, "Canary traffic increase is waiting for approval.", false, flaggerv1.SeverityWarn)
}
}
return false
} else {
if canary.Status.Phase == flaggerv1.CanaryPhaseWaitingTrafficIncrease {
if err := canaryController.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil {
c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
return false
}
}
}
c.recordEventInfof(canary, "Confirm-traffic-increase check %s passed", webhook.Name)
}
Expand Down