Skip to content

Commit

Permalink
report blocking by BZ component
Browse files Browse the repository at this point in the history
  • Loading branch information
@deads2k
deads2k committed Aug 31, 2020
1 parent 792761b commit 953f785
Show file tree
Hide file tree
Showing 3 changed files with 343 additions and 39 deletions.
82 changes: 59 additions & 23 deletions main.go
View file
Expand Up @@ -395,15 +395,41 @@ func updateGlobalBugCache(failedTestNames sets.String) {

func (a *Analyzer) identifyJobRunBugzillaFailer() {
for jrrKey, jrr := range a.RawData.JobRuns {
failingBugzillaComponents := sets.NewString()
if jrr.Succeeded { // we only care about failures.
continue
}

failingBugzillaComponents := map[string]*util.BugzillaComponentFailure{}

for _, failedTest := range jrr.FailedTestNames {
// If the failed test has a bug associated, use that bug to identify the BZ component.
// some tests are skipped
switch {
case strings.HasPrefix(failedTest, util.OperatorInstallPrefix) || strings.HasPrefix(failedTest, util.OperatorUpgradePrefix):
// these are handled by checking operator, no work on the string.
continue
case util.IgnoreTestRegex.MatchString(failedTest): // this test has no predictive value, do not assign
// If the failed test has a bug associated, use that bug to identify the BZ component.
continue
case failedTest == "Overall":
// this test has no predictive power. it just means the run failed.
continue
}

// Each bug that matches has its BZ component added.
foundBZ := false
for _, bug := range util.TestBugCache[failedTest] {
matchedRelease := false
for _, bugRelease := range bug.TargetRelease {
if strings.HasPrefix(bugRelease, a.Release) {
matchedRelease = true
break
}
}
if !matchedRelease {
continue
}
bzComponent := bug.Component[0]
failingBugzillaComponents.Insert(bzComponent)
addFailingBZComponent(&failingBugzillaComponents, bzComponent, failedTest, bug)
foundBZ = true

if result, found := a.RawData.ByAll["all"].TestResults[failedTest]; found {
Expand All @@ -415,29 +441,20 @@ func (a *Analyzer) identifyJobRunBugzillaFailer() {
}

// If we didn't have a bug, use the test name itself to identify a likely victim/blame
switch {
case strings.HasPrefix(failedTest, util.OperatorInstallPrefix) || strings.HasPrefix(failedTest, util.OperatorUpgradePrefix):
// these are handled by checking operator, no work on the string.
case failedTest == "Overall": // this test has no predictive value, do not assign

// TODO may have to add a case to cover all non-predictive tests

default:
bzComponent := util.GetBugzillaComponentForSig(util.FindSig(failedTest))
failingBugzillaComponents.Insert(bzComponent)
if result, found := a.RawData.ByAll["all"].TestResults[failedTest]; found {
util.SetTestResult(bzComponent, a.RawData.ByBugzillaComponent, failedTest, result.Successes, result.Failures, result.Flakes)
}
bzComponent := util.GetBugzillaComponentForSig(util.FindSig(failedTest))
addFailingBZComponent(&failingBugzillaComponents, bzComponent, failedTest, util.Bug{})
if result, found := a.RawData.ByAll["all"].TestResults[failedTest]; found {
util.SetTestResult(bzComponent, a.RawData.ByBugzillaComponent, failedTest, result.Successes, result.Failures, result.Flakes)
}
}
for _, operator := range jrr.InstallOperators {
if operator.State == util.Success {
continue
}
bzComponent := util.GetBugzillaComponentForOperator(operator.Name)
failingBugzillaComponents.Insert(bzComponent)

failedTest := util.OperatorInstallPrefix + operator.Name
addFailingBZComponent(&failingBugzillaComponents, bzComponent, failedTest, util.Bug{})

if result, found := a.RawData.ByAll["all"].TestResults[failedTest]; found {
util.SetTestResult(bzComponent, a.RawData.ByBugzillaComponent, failedTest, result.Successes, result.Failures, result.Flakes)
}
Expand All @@ -447,23 +464,42 @@ func (a *Analyzer) identifyJobRunBugzillaFailer() {
continue
}
bzComponent := util.GetBugzillaComponentForOperator(operator.Name)
failingBugzillaComponents.Insert(bzComponent)

failedTest := util.OperatorUpgradePrefix + operator.Name
addFailingBZComponent(&failingBugzillaComponents, bzComponent, failedTest, util.Bug{})

if result, found := a.RawData.ByAll["all"].TestResults[failedTest]; found {
util.SetTestResult(bzComponent, a.RawData.ByBugzillaComponent, failedTest, result.Successes, result.Failures, result.Flakes)
}
}

if jrr.SetupStatus != util.Success {
failingBugzillaComponents.Insert("Test Infrastructure")
if jrr.SetupStatus != util.Success && len(jrr.InstallOperators) == 0 { // we only want to count it as an infra issue if the install did not start
addFailingBZComponent(&failingBugzillaComponents, "Test Infrastructure", "setup", util.Bug{})
}

jrr.FailingBugzillaComponents = failingBugzillaComponents.List()
jrr.FailingBugzillaComponents = failingBugzillaComponents
a.RawData.JobRuns[jrrKey] = jrr
}
}

func addFailingBZComponent(failingBugzillaComponents *map[string]*util.BugzillaComponentFailure, bzComponent, testName string, bug util.Bug) {
failure, ok := (*failingBugzillaComponents)[bzComponent]
if !ok {
failure = &util.BugzillaComponentFailure{
ComponentName: bzComponent,
Failures: map[util.BugzillaTestFailure]int32{},
}
}

testFailure := util.BugzillaTestFailure{
TestName: testName,
BugID: bug.ID,
BugSummary: bug.Summary,
}
failure.Failures[testFailure] = failure.Failures[testFailure] + 1

(*failingBugzillaComponents)[bzComponent] = failure
}

func (a *Analyzer) analyze() {
for _, details := range a.RawData.JobDetails {
klog.V(2).Infof("processing test details for job %s\n", details.Name)
Expand Down
219 changes: 209 additions & 10 deletions pkg/html/html.go
View file
Expand Up @@ -100,7 +100,8 @@ Data current as of: %s
<a href="#TopFailingTestsWithABug">Top Failing Tests With a Bug</a> |
<a href="#JobPassRatesByJobName">Job Pass Rates By Job Name</a> | <a href="#CanaryTestFailures">Canary Test Failures</a> |
<a href="#JobRunsWithFailureGroups">Job Runs With Failure Groups</a> | <a href="#TestImpactingBugs">Test Impacting Bugs</a> |
<a href="#TestImpactingComponents">Test Impacting Components</a>
<a href="#TestImpactingComponents">Test Impacting Components</a> |
<a href="#BZToWorstJobImpact">Bugzilla Components to Worst Job Impact</a>
</p>
{{ summaryAcrossAllJobs .Current.All .Prev.All .EndDay }}
Expand All @@ -121,6 +122,8 @@ Data current as of: %s
{{ testImpactingComponents .Current.BugsByFailureCount }}
{{ summaryJobsFailuresByBugzillaComponent .Current .Prev .EndDay .JobTestCount .Release }}
`

// 1 encoded job name
Expand Down Expand Up @@ -347,6 +350,201 @@ func summaryJobsByPlatform(report, reportPrev util.TestReport, endDay, jobTestCo
return s
}

func summaryJobsFailuresByBugzillaComponent(report, reportPrev util.TestReport, endDay, jobTestCount int, release string) string {
failuresByBugzillaComponent := util.SummarizeJobsFailuresByBugzillaComponent(report)
failuresByBugzillaComponentPrev := util.SummarizeJobsFailuresByBugzillaComponent(reportPrev)

s := fmt.Sprintf(`
<table class="table">
<tr>
<th colspan=4 class="text-center"><a class="text-dark" title="Bugzilla components ranked by maximum fail percentage of any job." id="BZToWorstJobImpact" href="#BZToWorstJobImpact">Bugzilla Components to Worst Job Impact</a></th>
</tr>
<tr>
<th>Variant</th><th>Latest %d days</th><th/><th>Previous 7 days</th>
</tr>
`, endDay)

bzGroupTemplate := `
<tr class="%s">
<td>
%[2]s
<p>
<button class="btn btn-primary btn-sm py-0" style="font-size: 0.8em" type="button" data-toggle="collapse" data-target=".%[3]s" aria-expanded="false" aria-controls="%[3]s">Expand Failing Jobs</button>
</td>
<td>
%0.2f%% <span class="text-nowrap">(%d runs)</span>
</td>
<td>
%s
</td>
<td>
%0.2f%% <span class="text-nowrap">(%d runs)</span>
</td>
</tr>
`

naTemplate := `
<tr class="%s">
<td>
%[2]s
<p>
<button class="btn btn-primary btn-sm py-0" style="font-size: 0.8em" type="button" data-toggle="collapse" data-target=".%[3]s" aria-expanded="false" aria-controls="%[3]s">Expand Failing Jobs</button>
</td>
<td>
%0.2f%% <span class="text-nowrap">(%d runs)</span>
</td>
<td/>
<td>
NA
</td>
</tr>
`

bzFailingJobTemplate := `
<tr class="collapse %s">
<td>
%[2]s -- %[3]s
<p>
<button class="btn btn-primary btn-sm py-0" style="font-size: 0.8em" type="button" data-toggle="collapse" data-target=".%[4]s" aria-expanded="false" aria-controls="%[4]s">Expand Failing Tests</button>
</td>
<td>
%0.2f%% <span class="text-nowrap">(%d/%d failed/total)</span>
</td>
<td/>
<td>
NA
</td>
</tr>
`

for _, v := range failuresByBugzillaComponent {
safeBZJob := fmt.Sprintf("%s---component", v.Name)
safeBZJob = strings.ReplaceAll(safeBZJob, ".", "")
safeBZJob = strings.ReplaceAll(safeBZJob, " ", "")

prev := util.GetPrevBugzillaJobFailures(v.Name, failuresByBugzillaComponentPrev)
highestFailPercentage := v.JobsFailed[0].FailPercentageByThisBugzillaComponent
lowestPassPercentage := 100 - highestFailPercentage
rowColor := ""
switch {
case lowestPassPercentage > 85:
rowColor = "table-success"
case lowestPassPercentage > 70:
rowColor = "table-warning"
case lowestPassPercentage > 0:
rowColor = "table-danger"
default:
rowColor = "error"
}

if prev != nil {
previousHighestFailPercentage := prev.JobsFailed[0].FailPercentageByThisBugzillaComponent
previousLowestPassPercentage := 100 - previousHighestFailPercentage
arrow := ""

delta := 5.0
if v.JobsFailed[0].TotalRuns > 80 {
delta = 2
}
if lowestPassPercentage > previousLowestPassPercentage+delta {
arrow = up
} else if lowestPassPercentage < previousLowestPassPercentage-delta {
arrow = down
}

if lowestPassPercentage > previousLowestPassPercentage+delta {
arrow = fmt.Sprintf(up, lowestPassPercentage-previousLowestPassPercentage)
} else if lowestPassPercentage < previousLowestPassPercentage-delta {
arrow = fmt.Sprintf(down, previousLowestPassPercentage-lowestPassPercentage)
} else if lowestPassPercentage > previousLowestPassPercentage {
arrow = fmt.Sprintf(flatup, lowestPassPercentage-previousLowestPassPercentage)
} else {
arrow = fmt.Sprintf(flatdown, previousLowestPassPercentage-lowestPassPercentage)
}

s = s + fmt.Sprintf(bzGroupTemplate,
rowColor,
v.Name,
safeBZJob,
lowestPassPercentage,
-1,
arrow,
previousLowestPassPercentage,
-1,
)
} else {
s = s + fmt.Sprintf(naTemplate,
rowColor,
v.Name,
safeBZJob,
lowestPassPercentage,
-1,
)
}

count := 0
for _, failingJob := range v.JobsFailed {
bzJobTuple := fmt.Sprintf("%s---%s", v.Name, failingJob.JobName)
bzJobTuple = strings.ReplaceAll(bzJobTuple, ".", "")
bzJobTuple = strings.ReplaceAll(bzJobTuple, " ", "")

// only show the worst ten jobs
if count > 10 {
break
}
count++
s = s + fmt.Sprintf(bzFailingJobTemplate,
safeBZJob,
v.Name,
failingJob.JobName,
bzJobTuple,
100.0-failingJob.FailPercentageByThisBugzillaComponent,
failingJob.FailuresByThisBugzillaComponent,
failingJob.TotalRuns,
)

rows := ""
for info, count := range failingJob.Failures.Failures {
encodedTestName := url.QueryEscape(regexp.QuoteMeta(info.TestName))
jobQuery := fmt.Sprintf("%s", report.Release)

bug := "Associated Bugs: "
bugList := util.TestBugCache[info.TestName]
for _, b := range bugList {
for _, r := range b.TargetRelease {
if strings.HasPrefix(r, report.Release) {
bug += fmt.Sprintf("<a target=\"_blank\" href=%s>%d</a> ", b.Url, b.ID)
break
}
}
}
if len(bugList) == 0 {
bug = `<a style="padding-left: 20px" target="_blank" href="https://search.ci.openshift.org/?context=1&type=bug&maxMatches=5&maxBytes=20971520&groupBy=job&search=%s">Bug Search</a>`
bug = fmt.Sprintf(bug, encodedTestName)
}

rows = rows + fmt.Sprintf(testGroupTemplate, bzJobTuple,
info.TestName,
jobQuery,
encodedTestName,
bug,
100.0-(float32(count)/float32(failingJob.TotalRuns)*100.0),
failingJob.TotalRuns,
)
}

if len(rows) > 0 {
s = s + fmt.Sprintf(`<tr class="collapse %s"><td colspan=2 class="font-weight-bold">Test Name</td><td class="font-weight-bold">Test Pass Rate</td></tr>`, bzJobTuple)
s = s + rows
} else {
s = s + fmt.Sprintf(`<tr class="collapse %s"><td colspan=3 class="font-weight-bold">No Tests Matched Filters</td></tr>`, bzJobTuple)
}
}
}
s = s + "</table>"
return s
}

func openABugHTML(testName, release string) string {
short_desc := testName
if len(short_desc) > 255 {
Expand Down Expand Up @@ -818,15 +1016,16 @@ func PrintHtmlReport(w http.ResponseWriter, req *http.Request, report, prevRepor

var dashboardPage = template.Must(template.New("dashboardPage").Funcs(
template.FuncMap{
"summaryAcrossAllJobs": summaryAcrossAllJobs,
"failureGroups": failureGroups,
"summaryJobsByPlatform": summaryJobsByPlatform,
"summaryTopFailingTests": summaryTopFailingTests,
"summaryJobPassRatesByJobName": summaryJobPassRatesByJobName,
"canaryTestFailures": canaryTestFailures,
"failureGroupList": failureGroupList,
"testImpactingBugs": testImpactingBugs,
"testImpactingComponents": testImpactingComponents,
"summaryAcrossAllJobs": summaryAcrossAllJobs,
"failureGroups": failureGroups,
"summaryJobsByPlatform": summaryJobsByPlatform,
"summaryTopFailingTests": summaryTopFailingTests,
"summaryJobPassRatesByJobName": summaryJobPassRatesByJobName,
"canaryTestFailures": canaryTestFailures,
"failureGroupList": failureGroupList,
"testImpactingBugs": testImpactingBugs,
"testImpactingComponents": testImpactingComponents,
"summaryJobsFailuresByBugzillaComponent": summaryJobsFailuresByBugzillaComponent,
},
).Parse(dashboardPageHtml))

Expand Down

0 comments on commit 953f785

Please sign in to comment.