Skip to content

Commit

Permalink
Add system code to aggregate healtcheck response
Browse files Browse the repository at this point in the history
Modify html template
Fix typos
  • Loading branch information
dtvalk-ov committed Jan 13, 2022
1 parent b8b1381 commit 387e482
Show file tree
Hide file tree
Showing 7 changed files with 136 additions and 35 deletions.
39 changes: 33 additions & 6 deletions controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ type healthCheckController struct {
}

type controller interface {
buildServicesHealthResult([]string, bool) (fthealth.HealthResult, map[string]category, error)
buildServicesHealthResult([]string, bool) (enrichedHealthResult, map[string]category, error)
runServiceChecksByServiceNames(map[string]service, map[string]category) ([]fthealth.CheckResult, error)
runServiceChecksFor(map[string]category) ([]fthealth.CheckResult, error)
buildPodsHealthResult(string) (fthealth.HealthResult, error)
Expand All @@ -36,6 +36,17 @@ type controller interface {
getMeasuredServices() map[string]measuredService
}

//Both enriched structs are used for applying the services' system code to the ft check results
type enrichedHealthResult struct {
HealthResult fthealth.HealthResult
Checks []enrichedCheckResult
}

type enrichedCheckResult struct {
CheckResult fthealth.CheckResult
SystemCode string
}

func initializeController(environment string) *healthCheckController {
service := initializeHealthCheckService()
measuredServices := make(map[string]measuredService)
Expand Down Expand Up @@ -85,12 +96,12 @@ func (c *healthCheckController) addAck(serviceName string, ackMessage string) er
return nil
}

func (c *healthCheckController) buildServicesHealthResult(providedCategories []string, useCache bool) (fthealth.HealthResult, map[string]category, error) {
func (c *healthCheckController) buildServicesHealthResult(providedCategories []string, useCache bool) (enrichedHealthResult, map[string]category, error) {
var checkResults []fthealth.CheckResult
desc := "Health of the whole cluster of the moment served without cache."
availableCategories, err := c.healthCheckService.getCategories()
if err != nil {
return fthealth.HealthResult{}, nil, fmt.Errorf("cannot build health check result for services: %v", err.Error())
return enrichedHealthResult{}, nil, fmt.Errorf("cannot build health check result for services: %v", err.Error())
}

matchingCategories := getMatchingCategories(providedCategories, availableCategories)
Expand All @@ -102,7 +113,7 @@ func (c *healthCheckController) buildServicesHealthResult(providedCategories []s
checkResults, err = c.runServiceChecksFor(matchingCategories)
}
if err != nil {
return fthealth.HealthResult{}, nil, fmt.Errorf("cannot build health check result for services: %v", err.Error())
return enrichedHealthResult{}, nil, fmt.Errorf("cannot build health check result for services: %v", err.Error())
}

c.disableStickyFailingCategories(matchingCategories, checkResults)
Expand All @@ -111,17 +122,33 @@ func (c *healthCheckController) buildServicesHealthResult(providedCategories []s

health := fthealth.HealthResult{
SystemCode: c.environment,
Checks: checkResults,
Description: desc,
Name: c.environment + " cluster health",
SchemaVersion: 1,
Ok: finalOk,
Severity: finalSeverity,
}
enrichedCheckResults := c.enrichCheckResults(checkResults)
enrichedHealthResult := enrichedHealthResult{
HealthResult: health,
Checks: enrichedCheckResults,
}

sort.Sort(byNameComparator(health.Checks))

return health, matchingCategories, nil
return enrichedHealthResult, matchingCategories, nil
}

func (c *healthCheckController) enrichCheckResults(checkResults []fthealth.CheckResult) []enrichedCheckResult {
enrichedResults := make([]enrichedCheckResult, 0, len(checkResults))
for _, result := range checkResults {
enriched := enrichedCheckResult{
CheckResult: result,
SystemCode: c.healthCheckService.getServiceCodeByName(result.Name),
}
enrichedResults = append(enrichedResults, enriched)
}
return enrichedResults
}

func (c *healthCheckController) runServiceChecksByServiceNames(services map[string]service, categories map[string]category) ([]fthealth.CheckResult, error) {
Expand Down
18 changes: 14 additions & 4 deletions controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,32 @@ func (m *MockService) getServiceByName(serviceName string) (service, error) {
}
return service{
name: "test-service-name",
sysCode: "system-code",
ack: "test ack",
isResilient: strings.HasPrefix(serviceName, "resilient"),
}, m.getServiceByNameErr
}

func (m *MockService) getServiceCodeByName(serviceName string) string {
if serviceName == nonExistingServiceName {
return ""
}
return "system-code"
}

func (m *MockService) getServicesMapByNames(serviceNames []string) map[string]service {
if len(serviceNames) != 0 && serviceNames[0] == nonExistingServiceName {
return map[string]service{}
}
services := make(map[string]service)
services["test-service-name"] = service{
name: "test-service-name",
ack: "test ack",
name: "test-service-name",
sysCode: "system-code",
ack: "test ack",
}
services["test-service-name-2"] = service{
name: "test-service-name-2",
name: "test-service-name-2",
sysCode: "system-code-2",
}
return services
}
Expand Down Expand Up @@ -504,7 +514,7 @@ func TestRunServiceChecksForStickyCategoryUpdateError(t *testing.T) {
hc, categories, _ := controller.buildServicesHealthResult([]string{"test", "publishing", nonExistingCategoryName}, true)

assert.NotNil(t, hc)
assert.False(t, hc.Ok)
assert.False(t, hc.HealthResult.Ok)
assert.False(t, categories["test"].isEnabled)
}

Expand Down
34 changes: 20 additions & 14 deletions handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ type httpHandler struct {
//IndividualHealthcheckParams struct used to populate HTML template with individual checks
type IndividualHealthcheckParams struct {
Name string
SystemCode string
Status string
LastUpdated string
MoreInfoPath string
BizOpsPath string
AddOrRemoveAckPath string
AddOrRemoveAckPathName string
AckMessage string
Expand Down Expand Up @@ -54,6 +56,7 @@ const (
addAckMsgTemplatePath = "html-templates/add-ack-message-form-template.html"
healthcheckPath = "/__health"
jsonContentType = "application/json"
bizOpsPathPrefix = "https://biz-ops.in.ft.com/System/"
)

func handleResponseWriterErr(err error) {
Expand Down Expand Up @@ -191,11 +194,11 @@ func (h *httpHandler) handleServicesHealthCheck(w http.ResponseWriter, r *http.R

if r.Header.Get("Accept") == jsonContentType {
for i, serviceCheck := range healthResult.Checks {
serviceHealthcheckURL := getServiceHealthcheckURL(h.clusterURL, h.pathPrefix, serviceCheck.Name)
healthResult.Checks[i].TechnicalSummary = fmt.Sprintf("%s Service healthcheck: %s", serviceCheck.TechnicalSummary, serviceHealthcheckURL)
serviceHealthcheckURL := getServiceHealthcheckURL(h.clusterURL, h.pathPrefix, serviceCheck.CheckResult.Name)
healthResult.Checks[i].CheckResult.TechnicalSummary = fmt.Sprintf("%s Service healthcheck: %s", serviceCheck.CheckResult.TechnicalSummary, serviceHealthcheckURL)
}

buildHealthcheckJSONResponse(w, healthResult)
buildHealthcheckJSONResponse(w, healthResult.HealthResult)
} else {
env := h.controller.getEnvironment()
buildServicesCheckHTMLResponse(w, healthResult, env, getCategoriesString(validCategories), h.pathPrefix)
Expand Down Expand Up @@ -289,7 +292,7 @@ func (h *httpHandler) handleGoodToGo(w http.ResponseWriter, r *http.Request) {
}
}

if !healthResults.Ok {
if !healthResults.HealthResult.Ok {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
Expand Down Expand Up @@ -361,7 +364,7 @@ func buildHealthcheckJSONResponse(w http.ResponseWriter, healthResult fthealth.H
}
}

func buildServicesCheckHTMLResponse(w http.ResponseWriter, healthResult fthealth.HealthResult, environment string, categories string, pathPrefix string) {
func buildServicesCheckHTMLResponse(w http.ResponseWriter, healthResult enrichedHealthResult, environment string, categories string, pathPrefix string) {
w.Header().Add("Content-Type", "text/html")
htmlTemplate := parseHTMLTemplate(w, healthcheckTemplateName)
if htmlTemplate == nil {
Expand Down Expand Up @@ -410,15 +413,15 @@ func parseHTMLTemplate(w http.ResponseWriter, templateName string) *template.Tem
return htmlTemplate
}

func populateAggregateServiceChecks(healthResult fthealth.HealthResult, environment string, categories string, pathPrefix string) *AggregateHealthcheckParams {
indiviualServiceChecks, ackCount := populateIndividualServiceChecks(healthResult.Checks, pathPrefix)
func populateAggregateServiceChecks(healthResult enrichedHealthResult, environment string, categories string, pathPrefix string) *AggregateHealthcheckParams {
individualServiceChecks, ackCount := populateIndividualServiceChecks(healthResult.Checks, pathPrefix)
aggregateChecks := &AggregateHealthcheckParams{
PageTitle: buildPageTitle(environment, categories),
GeneralStatus: getGeneralStatus(healthResult),
GeneralStatus: getGeneralStatus(healthResult.HealthResult),
RefreshFromCachePath: buildRefreshFromCachePath(categories, pathPrefix),
RefreshWithoutCachePath: buildRefreshWithoutCachePath(categories, pathPrefix),
AckCount: ackCount,
IndividualHealthChecks: indiviualServiceChecks,
IndividualHealthChecks: individualServiceChecks,
}

return aggregateChecks
Expand All @@ -441,30 +444,33 @@ func buildRefreshWithoutCachePath(categories string, pathPrefix string) string {
return refreshWithoutCachePath
}

func populateIndividualServiceChecks(checks []fthealth.CheckResult, pathPrefix string) ([]IndividualHealthcheckParams, int) {
indiviualServiceChecks := make([]IndividualHealthcheckParams, len(checks))
func populateIndividualServiceChecks(checks []enrichedCheckResult, pathPrefix string) ([]IndividualHealthcheckParams, int) {
individualServiceChecks := make([]IndividualHealthcheckParams, len(checks))
ackCount := 0
for i, individualCheck := range checks {
for i, enrichedCheck := range checks {
individualCheck := enrichedCheck.CheckResult
if individualCheck.Ack != "" {
ackCount++
}

addOrRemoveAckPath, addOrRemoveAckPathName := buildAddOrRemoveAckPath(individualCheck.Name, pathPrefix, individualCheck.Ack)
hc := IndividualHealthcheckParams{
Name: individualCheck.Name,
SystemCode: enrichedCheck.SystemCode,
Status: getServiceStatusFromCheck(individualCheck),
LastUpdated: individualCheck.LastUpdated.Format(timeLayout),
MoreInfoPath: fmt.Sprintf("%s/__pods-health?service-name=%s", pathPrefix, individualCheck.Name),
BizOpsPath: fmt.Sprintf("%s%s", bizOpsPathPrefix, enrichedCheck.SystemCode),
AddOrRemoveAckPath: addOrRemoveAckPath,
AddOrRemoveAckPathName: addOrRemoveAckPathName,
AckMessage: individualCheck.Ack,
Output: individualCheck.CheckOutput,
}

indiviualServiceChecks[i] = hc
individualServiceChecks[i] = hc
}

return indiviualServiceChecks, ackCount
return individualServiceChecks, ackCount
}

func buildAddOrRemoveAckPath(serviceName string, pathPrefix string, ackMessage string) (string, string) {
Expand Down
21 changes: 13 additions & 8 deletions handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ func init() {
logger.InitLogger("upp-aggregate-healthcheck", "debug")
}

func (m *mockController) buildServicesHealthResult(providedCategories []string, useCache bool) (fthealth.HealthResult, map[string]category, error) {
func (m *mockController) buildServicesHealthResult(providedCategories []string, useCache bool) (enrichedHealthResult, map[string]category, error) {
if len(providedCategories) == 1 && providedCategories[0] == brokenCategoryName {
return fthealth.HealthResult{}, map[string]category{}, errors.New("Broken category")
return enrichedHealthResult{}, map[string]category{}, errors.New("Broken category")
}

matchingCategories := map[string]category{}
Expand All @@ -51,31 +51,36 @@ func (m *mockController) buildServicesHealthResult(providedCategories []string,
}
}

var checks []fthealth.CheckResult
var checks []enrichedCheckResult
finalOk := true
if len(providedCategories) == 1 && providedCategories[0] == categoryWithChecks {
checks = []fthealth.CheckResult{
checks = []enrichedCheckResult{
{
Ok: true,
CheckResult: fthealth.CheckResult{Ok: true},
},
{
Ok: false,
CheckResult: fthealth.CheckResult{Ok: false},
},
}

finalOk = false
}

health := fthealth.HealthResult{
Checks: checks,
//Checks: checks,
Description: "test",
Name: "cluster health",
SchemaVersion: 1,
Ok: finalOk,
Severity: 1,
}

return health, matchingCategories, nil
enrichedHealth := enrichedHealthResult{
HealthResult: health,
Checks: checks,
}

return enrichedHealth, matchingCategories, nil
}

func (m *mockController) getMeasuredServices() map[string]measuredService {
Expand Down
2 changes: 2 additions & 0 deletions html-templates/healthcheck-template.html
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ <h1>{{.PageTitle}}
<thead>
<tr>
<th>Name</th>
<th>System Code</th>
<th>Health status</th>
<th>Output</th>
<th>Last Updated</th>
Expand All @@ -51,6 +52,7 @@ <h1>{{.PageTitle}}
{{range .}}
<tr>
<td><a href="{{.MoreInfoPath}}">{{.Name}}</a></td>
<td><a href="{{.BizOpsPath}}">{{.SystemCode}}</a></td>
<td>&nbsp;
{{if eq .Status "ok"}}
<span style='color: green;'>ok</span>
Expand Down
1 change: 1 addition & 0 deletions model.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type deployment struct {

type service struct {
name string
sysCode string
ack string
appPort int32
isResilient bool
Expand Down
Loading

0 comments on commit 387e482

Please sign in to comment.