Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

rollout: Include health report in service object #51

Merged
merged 7 commits into from
Aug 7, 2020
Merged

rollout: Include health report in service object #51

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
013b647
rollout: Include health report in service object
Aug 4, 2020
0d42934
Report in human-readable format
Aug 5, 2020
a1f4653
Add todo about JSONReport
Aug 5, 2020
4e641f1
Rename last status to status
Aug 5, 2020
b7d207b
Fix typo
Aug 5, 2020
6da8e9e
Remove JSON report
Aug 6, 2020
52a0478
Create setAnnotation function
Aug 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions pkg/health/report.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,6 @@ import (
)

// StringReport returns a human-readable report of the diagnosis.
//
// The returned string has the format:
// status: healthy
// metrics:
// - request-latency[p99]: 500.00 (needs 750.0)
// - request-count: 800 (needs 1000)
func StringReport(healthCriteria []config.Metric, diagnosis Diagnosis) string {
report := fmt.Sprintf("status: %s\n", diagnosis.OverallResult.String())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what if we also added the time we wrote this down?
does the user have any way of telling when was this status report for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could, but I'd probably have a function setHealthReportAnnotation(report string) in rollout since there are cases in which this function is not called (e.g. new candidate). In that function, we can append the time at the end of any report.

I will defer this after #57 since we would need the mocked time introduced there. Doing it here would be double work and would make this PR much longer.

report += "metrics:"
Expand Down
34 changes: 22 additions & 12 deletions pkg/rollout/rollout.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,12 @@ func (r *Rollout) UpdateService(svc *run.Service) (*run.Service, error) {
if isNewCandidate(svc, candidate) {
r.log.Debug("new candidate, assign some traffic")
svc = r.PrepareRollForward(svc, stable, candidate)
svc = r.updateAnnotations(svc, stable, candidate, "new candidate, no health report available yet")
svc = r.updateAnnotations(svc, stable, candidate)

// TODO(gvso): create a setHealthReportAnnotation that appends the
// current time to the report before setting the annotation.
Copy link
Contributor

@ahmetb ahmetb Aug 7, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this probably could be part of the health report string generation IMO.
doesn't have to be here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My initial thought was also this one, but the problem is that we don't generate a report when we have a new candidate. We have a string literal in that case

setAnnotation(svc, LastHealthReportAnnotation, "new candidate, no health report available yet")

err := r.replaceService(svc)
return svc, errors.Wrap(err, "failed to replace service")
}
Expand All @@ -148,8 +153,10 @@ func (r *Rollout) UpdateService(svc *run.Service) (*run.Service, error) {
return nil, errors.Errorf("invalid candidate's health diagnosis %v", diagnosis.OverallResult)
}

svc = r.updateAnnotations(svc, stable, candidate)
report := health.StringReport(r.strategy.HealthCriteria, diagnosis)
svc = r.updateAnnotations(svc, stable, candidate, report)
setAnnotation(svc, LastHealthReportAnnotation, report)

err = r.replaceService(svc)
return svc, errors.Wrap(err, "failed to replace service")
}
Expand Down Expand Up @@ -286,28 +293,31 @@ func (r *Rollout) nextCandidateTraffic(current int64) int64 {
}

// updateAnnotations updates the annotations to keep some state about the rollout.
func (r *Rollout) updateAnnotations(svc *run.Service, stable, candidate, healthReport string) *run.Service {
if svc.Metadata.Annotations == nil {
svc.Metadata.Annotations = make(map[string]string)
}
svc.Metadata.Annotations[LastHealthReportAnnotation] = healthReport

func (r *Rollout) updateAnnotations(svc *run.Service, stable, candidate string) *run.Service {
// The candidate has become the stable revision.
if r.promoteToStable {
svc.Metadata.Annotations[StableRevisionAnnotation] = candidate
setAnnotation(svc, StableRevisionAnnotation, candidate)
delete(svc.Metadata.Annotations, CandidateRevisionAnnotation)
return svc
}

svc.Metadata.Annotations[StableRevisionAnnotation] = stable
svc.Metadata.Annotations[CandidateRevisionAnnotation] = candidate
setAnnotation(svc, StableRevisionAnnotation, stable)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice, yeah you can just use it here :D

setAnnotation(svc, CandidateRevisionAnnotation, candidate)
if r.shouldRollback {
svc.Metadata.Annotations[LastFailedCandidateRevisionAnnotation] = candidate
setAnnotation(svc, LastFailedCandidateRevisionAnnotation, candidate)
}

return svc
}

// setAnnotation sets the value of an annotation.
func setAnnotation(svc *run.Service, key, value string) {
if svc.Metadata.Annotations == nil {
svc.Metadata.Annotations = make(map[string]string)
}
svc.Metadata.Annotations[key] = value
}

// diagnoseCandidate returns the candidate's diagnosis based on metrics.
func (r *Rollout) diagnoseCandidate(candidate string, healthCriteria []config.Metric) (d health.Diagnosis, err error) {
healthCheckOffset := time.Duration(r.strategy.HealthOffsetMinute) * time.Minute
Expand Down