Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat (engine,cdsctl): improve global status compute #2582

Merged
merged 2 commits into from
Apr 18, 2018
Merged

feat (engine,cdsctl): improve global status compute #2582

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ba1455
feat (engine,cdsctl): improve global status compute
yesnault Apr 17, 2018
6409502
cr
yesnault Apr 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
63 changes: 8 additions & 55 deletions cli/cdsctl/monitoring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ type Termui struct {
// monitoring
queue *cli.ScrollableList
statusHatcheriesWorkers *cli.ScrollableList
statusWorkerModels *cli.ScrollableList
status *cli.ScrollableList
currentURL string
}
Expand All @@ -57,7 +56,6 @@ type Termui struct {
const (
QueueSelected = "queue"
BuildingSelected = "building"
WorkerModelsSelected = "workerModels"
HatcheriesWorkersSelected = "hatcheriesWorkers"
StatusSelected = "status"
)
Expand Down Expand Up @@ -176,27 +174,20 @@ func (ui *Termui) showMonitoring() {

ui.selected = QueueSelected

ui.statusWorkerModels = cli.NewScrollableList()
ui.statusWorkerModels.BorderLabel = " Worker Models "
ui.statusWorkerModels.Height = heightBottom
ui.statusWorkerModels.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.statusWorkerModels.BorderBottom = false
ui.statusWorkerModels.BorderLeft = false

ui.statusHatcheriesWorkers = cli.NewScrollableList()
ui.statusHatcheriesWorkers.BorderLabel = " Hatcheries "
ui.statusHatcheriesWorkers.Height = heightBottom
ui.statusHatcheriesWorkers.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.statusHatcheriesWorkers.BorderBottom = false
ui.statusHatcheriesWorkers.BorderLeft = false
ui.statusHatcheriesWorkers.BorderLeft = true
ui.statusHatcheriesWorkers.BorderRight = false

ui.status = cli.NewScrollableList()
ui.status.BorderLabel = " Status "
ui.status.Height = heightBottom
ui.status.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.status.BorderBottom = false
ui.status.BorderLeft = true
ui.status.BorderLeft = false
ui.status.BorderRight = false

termui.Body.AddRows(
Expand All @@ -216,9 +207,8 @@ func (ui *Termui) showMonitoring() {
)
termui.Body.AddRows(
termui.NewRow(
termui.NewCol(4, 0, ui.statusWorkerModels),
termui.NewCol(7, 0, ui.status),
termui.NewCol(5, 0, ui.statusHatcheriesWorkers),
termui.NewCol(3, 0, ui.status),
),
)

Expand Down Expand Up @@ -254,8 +244,6 @@ func (ui *Termui) monitoringCursorDown() {
switch ui.selected {
case QueueSelected:
ui.queue.CursorDown()
case WorkerModelsSelected:
ui.statusWorkerModels.CursorDown()
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.CursorDown()
case StatusSelected:
Expand All @@ -267,8 +255,6 @@ func (ui *Termui) monitoringCursorUp() {
switch ui.selected {
case QueueSelected:
ui.queue.CursorUp()
case WorkerModelsSelected:
ui.statusWorkerModels.CursorUp()
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.CursorUp()
case StatusSelected:
Expand All @@ -282,9 +268,6 @@ func (ui *Termui) monitoringSelectNext() {
case QueueSelected:
ui.selected = BuildingSelected
ui.queue.Cursor = 0
case WorkerModelsSelected:
ui.selected = HatcheriesWorkersSelected
ui.statusWorkerModels.Cursor = 0
case HatcheriesWorkersSelected:
ui.selected = StatusSelected
ui.statusHatcheriesWorkers.Cursor = 0
Expand All @@ -297,22 +280,18 @@ func (ui *Termui) monitoringSelectNext() {

func (ui *Termui) monitoringColorSelected() {
ui.queue.BorderFg = termui.ColorDefault
ui.statusWorkerModels.BorderFg = termui.ColorDefault
ui.statusHatcheriesWorkers.BorderFg = termui.ColorDefault
ui.status.BorderFg = termui.ColorDefault

switch ui.selected {
case QueueSelected:
ui.queue.BorderFg = termui.ColorRed
case WorkerModelsSelected:
ui.statusWorkerModels.BorderFg = termui.ColorRed
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.BorderFg = termui.ColorRed
case StatusSelected:
ui.status.BorderFg = termui.ColorRed
}
termui.Render(ui.queue,
ui.statusWorkerModels,
ui.statusHatcheriesWorkers,
ui.status)
}
Expand All @@ -327,17 +306,12 @@ func (ui *Termui) updateStatus() string {
elapsed := time.Since(start)
msg := fmt.Sprintf("[status %s](fg-cyan,bg-default)", sdk.Round(elapsed, time.Millisecond).String())

selected := "fg-white,bg-default"
if ui.selected == StatusSelected {
selected = "fg-white"
}

items := []string{}
for _, l := range statusEngine.Lines {
if l.Status != sdk.MonitoringStatusOK {
items = append(items, fmt.Sprintf("[%s](bg-red)", l.String()))
} else if strings.Contains(l.Status, "Version") {
items = append(items, fmt.Sprintf("[%s](%s)", l.String(), selected))
items = append(items, fmt.Sprintf("[%s](fg-white,bg-red)", l.String()))
} else if strings.Contains(l.Component, "Global") {
items = append(items, fmt.Sprintf("[%s](fg-white,bg-default)", l.String()))
}
}
ui.status.Items = items
Expand Down Expand Up @@ -480,15 +454,14 @@ func (ui *Termui) computeStatusWorkerModels(workers []sdk.Worker) (string, map[i
wModels[workerModels[w].ID] = workerModels[w]
}

statusTitle, items, idsModels := []string{}, []string{}, []string{}
statusTitle := []string{}
models := make(map[string]map[string]int64)
status := make(map[string]int)

for _, w := range workers {
idModel := fmt.Sprintf("%s", wModels[w.ModelID].Type+" "+wModels[w.ModelID].Name)
if _, ok := models[idModel]; !ok {
models[idModel] = make(map[string]int64)
idsModels = append(idsModels, idModel)
}
models[idModel][w.Status.String()] = models[idModel][w.Status.String()] + 1

Expand All @@ -498,33 +471,13 @@ func (ui *Termui) computeStatusWorkerModels(workers []sdk.Worker) (string, map[i
status[w.Status.String()] = status[w.Status.String()] + 1
}

selected := ",bg-default"
if ui.selected == WorkerModelsSelected {
selected = ""
}

sort.Strings(idsModels)
for _, id := range idsModels {
v := models[id]
var t string
for _, status := range statusTitle {
if v[status] > 0 {
icon, color := statusShort(status)
t += fmt.Sprintf("[%d %s ](%s%s)", v[status], icon, color, selected)
}
}
t += fmt.Sprintf("[ %s](fg-white%s)", pad(id, 28), selected)
items = append(items, t)
}
ui.statusWorkerModels.Items = items

sort.Strings(statusTitle)
title := " Models "
for _, s := range statusTitle {
icon, color := statusShort(s)
title += fmt.Sprintf("[%d %s](%s) ", status[s], icon, color)
}
ui.statusWorkerModels.BorderLabel = title
ui.workers.Text += " " + title

return msg, wModels
}
Expand Down
69 changes: 68 additions & 1 deletion engine/api/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"runtime"
"sort"
"strings"

"github.com/ovh/cds/engine/api/event"
Expand Down Expand Up @@ -76,22 +77,48 @@ func (api *API) statusHandler() Handler {
}
}

type computeGlobalNumbers struct {
nbSrv int
nbOK int
nbAlerts int
nbWarn int
}

func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
mStatus := sdk.MonitoringStatus{}

var version string
versionOk := true
linesGlobal := []sdk.MonitoringStatusLine{}

resume := map[string]computeGlobalNumbers{
services.TypeAPI: computeGlobalNumbers{},
services.TypeRepositories: computeGlobalNumbers{},
services.TypeVCS: computeGlobalNumbers{},
services.TypeHooks: computeGlobalNumbers{},
services.TypeHatchery: computeGlobalNumbers{},
}
var nbg computeGlobalNumbers
for _, s := range srvs {
var nbOK, nbWarn, nbAlert int
for i := range s.MonitoringStatus.Lines {
l := s.MonitoringStatus.Lines[i]
mStatus.Lines = append(mStatus.Lines, l)

switch l.Status {
case sdk.MonitoringStatusOK:
nbOK++
case sdk.MonitoringStatusWarn:
nbWarn++
default:
nbAlert++
}

// services should have same version
if strings.Contains(l.Component, "Version") {
if version == "" {
version = l.Value
} else if version != l.Value {
} else if version != l.Value && versionOk {
versionOk = false
linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: sdk.MonitoringStatusWarn,
Expand All @@ -101,6 +128,18 @@ func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
}
}
}

t := resume[s.Type]
t.nbOK += nbOK
t.nbWarn += nbWarn
t.nbAlerts += nbAlert
t.nbSrv++
resume[s.Type] = t

nbg.nbOK += nbOK
nbg.nbWarn += nbWarn
nbg.nbAlerts += nbAlert
nbg.nbSrv++
}

if versionOk {
Expand All @@ -111,10 +150,38 @@ func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
})
}

linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: api.computeGlobalStatusByNumbers(nbg),
Component: "Global/Status",
Value: fmt.Sprintf("%d services", len(srvs)),
})

for stype, r := range resume {
linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: api.computeGlobalStatusByNumbers(r),
Component: fmt.Sprintf("Global/%s", stype),
Value: fmt.Sprintf("%d", r.nbSrv),
})
}

sort.Slice(linesGlobal, func(i, j int) bool {
return linesGlobal[i].Component < linesGlobal[j].Component
})

mStatus.Lines = append(linesGlobal, mStatus.Lines...)
return mStatus
}

func (api *API) computeGlobalStatusByNumbers(s computeGlobalNumbers) string {
r := sdk.MonitoringStatusOK
if s.nbAlerts > 0 {
r = sdk.MonitoringStatusAlert
} else if s.nbWarn > 0 {
r = sdk.MonitoringStatusWarn
}
return r
}

func (api *API) smtpPingHandler() Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if getUser(ctx) == nil {
Expand Down