Skip to content

Commit

Permalink
feat (engine,cdsctl): improve global status compute (#2582)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yesnault @sguiheux
yesnault authored and sguiheux committed Apr 18, 2018
1 parent 6ca2f70 commit 1dcc3a9
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 56 deletions.
63 changes: 8 additions & 55 deletions cli/cdsctl/monitoring.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ type Termui struct {
// monitoring
queue *cli.ScrollableList
statusHatcheriesWorkers *cli.ScrollableList
statusWorkerModels *cli.ScrollableList
status *cli.ScrollableList
currentURL string
}
Expand All @@ -57,7 +56,6 @@ type Termui struct {
const (
QueueSelected = "queue"
BuildingSelected = "building"
WorkerModelsSelected = "workerModels"
HatcheriesWorkersSelected = "hatcheriesWorkers"
StatusSelected = "status"
)
Expand Down Expand Up @@ -176,27 +174,20 @@ func (ui *Termui) showMonitoring() {

ui.selected = QueueSelected

ui.statusWorkerModels = cli.NewScrollableList()
ui.statusWorkerModels.BorderLabel = " Worker Models "
ui.statusWorkerModels.Height = heightBottom
ui.statusWorkerModels.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.statusWorkerModels.BorderBottom = false
ui.statusWorkerModels.BorderLeft = false

ui.statusHatcheriesWorkers = cli.NewScrollableList()
ui.statusHatcheriesWorkers.BorderLabel = " Hatcheries "
ui.statusHatcheriesWorkers.Height = heightBottom
ui.statusHatcheriesWorkers.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.statusHatcheriesWorkers.BorderBottom = false
ui.statusHatcheriesWorkers.BorderLeft = false
ui.statusHatcheriesWorkers.BorderLeft = true
ui.statusHatcheriesWorkers.BorderRight = false

ui.status = cli.NewScrollableList()
ui.status.BorderLabel = " Status "
ui.status.Height = heightBottom
ui.status.Items = []string{"[loading...](fg-cyan,bg-default)"}
ui.status.BorderBottom = false
ui.status.BorderLeft = true
ui.status.BorderLeft = false
ui.status.BorderRight = false

termui.Body.AddRows(
Expand All @@ -216,9 +207,8 @@ func (ui *Termui) showMonitoring() {
)
termui.Body.AddRows(
termui.NewRow(
termui.NewCol(4, 0, ui.statusWorkerModels),
termui.NewCol(7, 0, ui.status),
termui.NewCol(5, 0, ui.statusHatcheriesWorkers),
termui.NewCol(3, 0, ui.status),
),
)

Expand Down Expand Up @@ -254,8 +244,6 @@ func (ui *Termui) monitoringCursorDown() {
switch ui.selected {
case QueueSelected:
ui.queue.CursorDown()
case WorkerModelsSelected:
ui.statusWorkerModels.CursorDown()
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.CursorDown()
case StatusSelected:
Expand All @@ -267,8 +255,6 @@ func (ui *Termui) monitoringCursorUp() {
switch ui.selected {
case QueueSelected:
ui.queue.CursorUp()
case WorkerModelsSelected:
ui.statusWorkerModels.CursorUp()
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.CursorUp()
case StatusSelected:
Expand All @@ -282,9 +268,6 @@ func (ui *Termui) monitoringSelectNext() {
case QueueSelected:
ui.selected = BuildingSelected
ui.queue.Cursor = 0
case WorkerModelsSelected:
ui.selected = HatcheriesWorkersSelected
ui.statusWorkerModels.Cursor = 0
case HatcheriesWorkersSelected:
ui.selected = StatusSelected
ui.statusHatcheriesWorkers.Cursor = 0
Expand All @@ -297,22 +280,18 @@ func (ui *Termui) monitoringSelectNext() {

func (ui *Termui) monitoringColorSelected() {
ui.queue.BorderFg = termui.ColorDefault
ui.statusWorkerModels.BorderFg = termui.ColorDefault
ui.statusHatcheriesWorkers.BorderFg = termui.ColorDefault
ui.status.BorderFg = termui.ColorDefault

switch ui.selected {
case QueueSelected:
ui.queue.BorderFg = termui.ColorRed
case WorkerModelsSelected:
ui.statusWorkerModels.BorderFg = termui.ColorRed
case HatcheriesWorkersSelected:
ui.statusHatcheriesWorkers.BorderFg = termui.ColorRed
case StatusSelected:
ui.status.BorderFg = termui.ColorRed
}
termui.Render(ui.queue,
ui.statusWorkerModels,
ui.statusHatcheriesWorkers,
ui.status)
}
Expand All @@ -327,17 +306,12 @@ func (ui *Termui) updateStatus() string {
elapsed := time.Since(start)
msg := fmt.Sprintf("[status %s](fg-cyan,bg-default)", sdk.Round(elapsed, time.Millisecond).String())

selected := "fg-white,bg-default"
if ui.selected == StatusSelected {
selected = "fg-white"
}

items := []string{}
for _, l := range statusEngine.Lines {
if l.Status != sdk.MonitoringStatusOK {
items = append(items, fmt.Sprintf("[%s](bg-red)", l.String()))
} else if strings.Contains(l.Status, "Version") {
items = append(items, fmt.Sprintf("[%s](%s)", l.String(), selected))
items = append(items, fmt.Sprintf("[%s](fg-white,bg-red)", l.String()))
} else if strings.Contains(l.Component, "Global") {
items = append(items, fmt.Sprintf("[%s](fg-white,bg-default)", l.String()))
}
}
ui.status.Items = items
Expand Down Expand Up @@ -480,15 +454,14 @@ func (ui *Termui) computeStatusWorkerModels(workers []sdk.Worker) (string, map[i
wModels[workerModels[w].ID] = workerModels[w]
}

statusTitle, items, idsModels := []string{}, []string{}, []string{}
statusTitle := []string{}
models := make(map[string]map[string]int64)
status := make(map[string]int)

for _, w := range workers {
idModel := fmt.Sprintf("%s", wModels[w.ModelID].Type+" "+wModels[w.ModelID].Name)
if _, ok := models[idModel]; !ok {
models[idModel] = make(map[string]int64)
idsModels = append(idsModels, idModel)
}
models[idModel][w.Status.String()] = models[idModel][w.Status.String()] + 1

Expand All @@ -498,33 +471,13 @@ func (ui *Termui) computeStatusWorkerModels(workers []sdk.Worker) (string, map[i
status[w.Status.String()] = status[w.Status.String()] + 1
}

selected := ",bg-default"
if ui.selected == WorkerModelsSelected {
selected = ""
}

sort.Strings(idsModels)
for _, id := range idsModels {
v := models[id]
var t string
for _, status := range statusTitle {
if v[status] > 0 {
icon, color := statusShort(status)
t += fmt.Sprintf("[%d %s ](%s%s)", v[status], icon, color, selected)
}
}
t += fmt.Sprintf("[ %s](fg-white%s)", pad(id, 28), selected)
items = append(items, t)
}
ui.statusWorkerModels.Items = items

sort.Strings(statusTitle)
title := " Models "
for _, s := range statusTitle {
icon, color := statusShort(s)
title += fmt.Sprintf("[%d %s](%s) ", status[s], icon, color)
}
ui.statusWorkerModels.BorderLabel = title
ui.workers.Text += " " + title

return msg, wModels
}
Expand Down
69 changes: 68 additions & 1 deletion engine/api/status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"runtime"
"sort"
"strings"

"github.com/ovh/cds/engine/api/event"
Expand Down Expand Up @@ -76,22 +77,48 @@ func (api *API) statusHandler() Handler {
}
}

type computeGlobalNumbers struct {
nbSrv int
nbOK int
nbAlerts int
nbWarn int
}

func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
mStatus := sdk.MonitoringStatus{}

var version string
versionOk := true
linesGlobal := []sdk.MonitoringStatusLine{}

resume := map[string]computeGlobalNumbers{
services.TypeAPI: computeGlobalNumbers{},
services.TypeRepositories: computeGlobalNumbers{},
services.TypeVCS: computeGlobalNumbers{},
services.TypeHooks: computeGlobalNumbers{},
services.TypeHatchery: computeGlobalNumbers{},
}
var nbg computeGlobalNumbers
for _, s := range srvs {
var nbOK, nbWarn, nbAlert int
for i := range s.MonitoringStatus.Lines {
l := s.MonitoringStatus.Lines[i]
mStatus.Lines = append(mStatus.Lines, l)

switch l.Status {
case sdk.MonitoringStatusOK:
nbOK++
case sdk.MonitoringStatusWarn:
nbWarn++
default:
nbAlert++
}

// services should have same version
if strings.Contains(l.Component, "Version") {
if version == "" {
version = l.Value
} else if version != l.Value {
} else if version != l.Value && versionOk {
versionOk = false
linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: sdk.MonitoringStatusWarn,
Expand All @@ -101,6 +128,18 @@ func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
}
}
}

t := resume[s.Type]
t.nbOK += nbOK
t.nbWarn += nbWarn
t.nbAlerts += nbAlert
t.nbSrv++
resume[s.Type] = t

nbg.nbOK += nbOK
nbg.nbWarn += nbWarn
nbg.nbAlerts += nbAlert
nbg.nbSrv++
}

if versionOk {
Expand All @@ -111,10 +150,38 @@ func (api *API) computeGlobalStatus(srvs []sdk.Service) sdk.MonitoringStatus {
})
}

linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: api.computeGlobalStatusByNumbers(nbg),
Component: "Global/Status",
Value: fmt.Sprintf("%d services", len(srvs)),
})

for stype, r := range resume {
linesGlobal = append(linesGlobal, sdk.MonitoringStatusLine{
Status: api.computeGlobalStatusByNumbers(r),
Component: fmt.Sprintf("Global/%s", stype),
Value: fmt.Sprintf("%d", r.nbSrv),
})
}

sort.Slice(linesGlobal, func(i, j int) bool {
return linesGlobal[i].Component < linesGlobal[j].Component
})

mStatus.Lines = append(linesGlobal, mStatus.Lines...)
return mStatus
}

func (api *API) computeGlobalStatusByNumbers(s computeGlobalNumbers) string {
r := sdk.MonitoringStatusOK
if s.nbAlerts > 0 {
r = sdk.MonitoringStatusAlert
} else if s.nbWarn > 0 {
r = sdk.MonitoringStatusWarn
}
return r
}

func (api *API) smtpPingHandler() Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if getUser(ctx) == nil {
Expand Down

0 comments on commit 1dcc3a9

Please sign in to comment.