Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

separate the test result intermediate types from test result output types #66

Merged
merged 9 commits into from Sep 3, 2020
Merged

separate the test result intermediate types from test result output types #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cb9063e
Revert "report blocking by BZ component"
deads2k Aug 31, 2020
65fbf5a
Revert "add data for bz component sorting:"
deads2k Aug 31, 2020
b1fac27
Revert "add bz, operator, and sig mappings"
deads2k Aug 31, 2020
f728a18
create bug cache
deads2k Aug 31, 2020
dfe046f
move testgrid apis to api
deads2k Aug 31, 2020
cb0996a
stop producing output data dynamically
deads2k Aug 31, 2020
7ca81ad
ignore tests in one spot
deads2k Aug 31, 2020
f0ace23
straight move the remaining api serialization
deads2k Sep 1, 2020
5048cf7
separate the test result intermediate types from test result output t…
deads2k Sep 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
496 changes: 162 additions & 334 deletions main.go
View file
Open in desktop

Large diffs are not rendered by default.

169 changes: 57 additions & 112 deletions pkg/api/api.go
View file
Open in desktop
Expand Up @@ -7,76 +7,21 @@ import (
"net/url"
"regexp"

v12 "github.com/openshift/sippy/pkg/apis/sippyprocessing/v1"

v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"

"github.com/openshift/sippy/pkg/html"
"github.com/openshift/sippy/pkg/util"
"k8s.io/klog"
)

// PassRate describes statistics on a pass rate
type PassRate struct {
Percentage float64 `json:"percentage"`
ProjectedPercentage float64 `json:"projectedPercentage,omitempty"`
Runs int `json:"runs"`
}

// SummaryAcrossAllJobs describes the category summaryacrossalljobs
// valid keys are latest and prev
type SummaryAcrossAllJobs struct {
TestExecutions map[string]int `json:"testExecutions"`
TestPassPercentage map[string]float64 `json:"testPassPercentage"`
}

// FailureGroups describes the category failuregroups
// valid keys are latest and prev
type FailureGroups struct {
JobRunsWithFailureGroup map[string]int `json:"jobRunsWithFailureGroup"`
AvgFailureGroupSize map[string]int `json:"avgFailureGroupSize"`
MedianFailureGroupSize map[string]int `json:"medianFailureGroupSize"`
}

// CanaryTestFailInstance describes one single instance of a canary test failure
// passRate should have percentage (float64) and number of runs (int)
type CanaryTestFailInstance struct {
Name string `json:"name"`
Url string `json:"url"`
PassRate PassRate `json:"passRate"`
}

// PassRatesByJobName is responsible for the section job pass rates by job name
type PassRatesByJobName struct {
Name string `json:"name"`
Url string `json:"url"`
PassRates map[string]PassRate `json:"passRates"`
}

// FailingTestBug describes a single instance of failed test with bug or failed test without bug
// differs from failingtest in that it includes pass rates for previous days and latest days
type FailingTestBug struct {
Name string `json:"name"`
Url string `json:"url"`
PassRates map[string]PassRate `json:"passRates"`
Bugs []util.Bug `json:"bugs,omitempty"`
}

// JobSummaryPlatform describes a single platform and its associated jobs, their pass rates, and failing tests
type JobSummaryPlatform struct {
Platform string `json:"platform"`
PassRates map[string]PassRate `json:"passRates"`
}

// FailureGroup describes a single failure group - does not show the associated failed job names
type FailureGroup struct {
Job string `json:"job"`
Url string `json:"url"`
TestFailures int `json:"testFailures"`
}

// summary across all job
func summaryAcrossAllJobs(result, resultPrev map[string]util.SortedAggregateTestResult, endDay int) *SummaryAcrossAllJobs {
func summaryAcrossAllJobs(result, resultPrev map[string]v12.SortedAggregateTestsResult, endDay int) *v1.SummaryAcrossAllJobs {
all := result["all"]
allPrev := resultPrev["all"]

summary := SummaryAcrossAllJobs{
summary := v1.SummaryAcrossAllJobs{
TestExecutions: map[string]int{
"latest": all.Successes + all.Failures,
"prev": allPrev.Successes + allPrev.Failures,
Expand All @@ -91,11 +36,11 @@ func summaryAcrossAllJobs(result, resultPrev map[string]util.SortedAggregateTest
}

// stats on failure groups
func failureGroups(failureGroups, failureGroupsPrev []util.JobRunResult, endDay int) *FailureGroups {
func failureGroups(failureGroups, failureGroupsPrev []v12.JobRunResult, endDay int) *v1.FailureGroups {

_, _, median, medianPrev, avg, avgPrev := util.ComputeFailureGroupStats(failureGroups, failureGroupsPrev)

failureGroupStruct := FailureGroups{
failureGroupStruct := v1.FailureGroups{
JobRunsWithFailureGroup: map[string]int{
"latest": len(failureGroups),
"prev": len(failureGroupsPrev),
Expand All @@ -112,38 +57,38 @@ func failureGroups(failureGroups, failureGroupsPrev []util.JobRunResult, endDay
return &failureGroupStruct
}

func summaryJobsByPlatform(report, reportPrev util.TestReport, endDay, jobTestCount int) []JobSummaryPlatform {
func summaryJobsByPlatform(report, reportPrev v12.TestReport, endDay, jobTestCount int) []v1.JobSummaryPlatform {
jobsByPlatform := util.SummarizeJobsByPlatform(report)
jobsByPlatformPrev := util.SummarizeJobsByPlatform(reportPrev)

var jobSummariesByPlatform []JobSummaryPlatform
var jobSummariesByPlatform []v1.JobSummaryPlatform

for _, v := range jobsByPlatform {
prev := util.GetPrevPlatform(v.Platform, jobsByPlatformPrev)

var jobSummaryPlatform JobSummaryPlatform
var jobSummaryPlatform v1.JobSummaryPlatform

if prev != nil {
jobSummaryPlatform = JobSummaryPlatform{
jobSummaryPlatform = v1.JobSummaryPlatform{
Platform: v.Platform,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: v.PassPercentage,
ProjectedPercentage: v.PassPercentageWithKnownFailures,
Runs: v.Successes + v.Failures,
},
"prev": PassRate{
"prev": v1.PassRate{
Percentage: prev.PassPercentage,
ProjectedPercentage: prev.PassPercentageWithKnownFailures,
Runs: prev.Successes + prev.Failures,
},
},
}
} else {
jobSummaryPlatform = JobSummaryPlatform{
jobSummaryPlatform = v1.JobSummaryPlatform{
Platform: v.Platform,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: v.PassPercentage,
ProjectedPercentage: v.PassPercentageWithKnownFailures,
Runs: v.Successes + v.Failures,
Expand All @@ -158,9 +103,9 @@ func summaryJobsByPlatform(report, reportPrev util.TestReport, endDay, jobTestCo
}

// top failing tests with a bug
func summaryTopFailingTestsWithBug(topFailingTestsWithBug []*util.TestResult, resultPrev map[string]util.SortedAggregateTestResult, endDay int) []FailingTestBug {
func summaryTopFailingTestsWithBug(topFailingTestsWithBug []*v12.TestResult, resultPrev map[string]v12.SortedAggregateTestsResult, endDay int) []v1.FailingTestBug {

var topFailingTests []FailingTestBug
var topFailingTests []v1.FailingTestBug

allPrev := resultPrev["all"]

Expand All @@ -170,30 +115,30 @@ func summaryTopFailingTestsWithBug(topFailingTestsWithBug []*util.TestResult, re
testLink := fmt.Sprintf("%s%s", html.BugSearchUrl, encodedTestName)
testPrev := util.GetPrevTest(test.Name, allPrev.TestResults)

var failedTestWithBug FailingTestBug
var failedTestWithBug v1.FailingTestBug

if testPrev != nil {
failedTestWithBug = FailingTestBug{
failedTestWithBug = v1.FailingTestBug{
Name: test.Name,
Url: testLink,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: test.PassPercentage,
Runs: test.Successes + test.Failures,
},
"prev": PassRate{
"prev": v1.PassRate{
Percentage: testPrev.PassPercentage,
Runs: testPrev.Successes + test.Failures,
},
},
Bugs: test.BugList,
}
} else {
failedTestWithBug = FailingTestBug{
failedTestWithBug = v1.FailingTestBug{
Name: test.Name,
Url: testLink,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: test.PassPercentage,
Runs: test.Successes + test.Failures,
},
Expand All @@ -210,43 +155,43 @@ func summaryTopFailingTestsWithBug(topFailingTestsWithBug []*util.TestResult, re
}

// top failing tests without a bug
func summaryTopFailingTestsWithoutBug(topFailingTestsWithoutBug []*util.TestResult, resultPrev map[string]util.SortedAggregateTestResult, endDay int) []FailingTestBug {
func summaryTopFailingTestsWithoutBug(topFailingTestsWithoutBug []*v12.TestResult, resultPrev map[string]v12.SortedAggregateTestsResult, endDay int) []v1.FailingTestBug {

allPrev := resultPrev["all"]

var topFailingTests []FailingTestBug
var topFailingTests []v1.FailingTestBug

for _, test := range topFailingTestsWithoutBug {
encodedTestName := url.QueryEscape(regexp.QuoteMeta(test.Name))

testLink := fmt.Sprintf("%s%s", html.BugSearchUrl, encodedTestName)
testPrev := util.GetPrevTest(test.Name, allPrev.TestResults)

var failedTestWithoutBug FailingTestBug
var failedTestWithoutBug v1.FailingTestBug

if testPrev != nil {

failedTestWithoutBug = FailingTestBug{
failedTestWithoutBug = v1.FailingTestBug{
Name: test.Name,
Url: testLink,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: test.PassPercentage,
Runs: test.Successes + test.Failures,
},
"prev": PassRate{
"prev": v1.PassRate{
Percentage: testPrev.PassPercentage,
Runs: testPrev.Successes + testPrev.Failures,
},
},
}

} else {
failedTestWithoutBug = FailingTestBug{
failedTestWithoutBug = v1.FailingTestBug{
Name: test.Name,
Url: testLink,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: test.PassPercentage,
Runs: test.Successes + test.Failures,
},
Expand All @@ -259,41 +204,41 @@ func summaryTopFailingTestsWithoutBug(topFailingTestsWithoutBug []*util.TestResu
return topFailingTests
}

func summaryJobPassRatesByJobName(report, reportPrev util.TestReport, endDay, jobTestCount int) []PassRatesByJobName {
func summaryJobPassRatesByJobName(report, reportPrev v12.TestReport, endDay, jobTestCount int) []v1.PassRatesByJobName {

jobRunsByName := util.SummarizeJobsByName(report)
jobRunsByNamePrev := util.SummarizeJobsByName(reportPrev)

var passRatesSlice []PassRatesByJobName
var passRatesSlice []v1.PassRatesByJobName

for _, v := range jobRunsByName {
prev := util.GetPrevJob(v.Name, jobRunsByNamePrev)

var newJobPassRate PassRatesByJobName
var newJobPassRate v1.PassRatesByJobName

if prev != nil {
newJobPassRate = PassRatesByJobName{
newJobPassRate = v1.PassRatesByJobName{
Name: v.Name,
Url: v.TestGridUrl,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: v.PassPercentage,
ProjectedPercentage: v.PassPercentageWithKnownFailures,
Runs: v.Successes + v.Failures,
},
"prev": PassRate{
"prev": v1.PassRate{
Percentage: prev.PassPercentage,
ProjectedPercentage: prev.PassPercentageWithKnownFailures,
Runs: prev.Successes + prev.Failures,
},
},
}
} else {
newJobPassRate = PassRatesByJobName{
newJobPassRate = v1.PassRatesByJobName{
Name: v.Name,
Url: v.TestGridUrl,
PassRates: map[string]PassRate{
"latest": PassRate{
PassRates: map[string]v1.PassRate{
"latest": v1.PassRate{
Percentage: v.PassPercentage,
ProjectedPercentage: v.PassPercentageWithKnownFailures,
Runs: v.Successes + v.Failures,
Expand All @@ -311,10 +256,10 @@ func summaryJobPassRatesByJobName(report, reportPrev util.TestReport, endDay, jo
}

// canaryTestFailures section
func canaryTestFailures(result map[string]util.SortedAggregateTestResult) []CanaryTestFailInstance {
func canaryTestFailures(result map[string]v12.SortedAggregateTestsResult) []v1.CanaryTestFailInstance {
all := result["all"].TestResults

var canaryFailures []CanaryTestFailInstance
var canaryFailures []v1.CanaryTestFailInstance

if len(all) <= 0 {
return nil
Expand All @@ -324,10 +269,10 @@ func canaryTestFailures(result map[string]util.SortedAggregateTestResult) []Cana
test := all[i]
encodedTestName := url.QueryEscape(regexp.QuoteMeta(test.Name))
canaryFailures = append(canaryFailures,
CanaryTestFailInstance{
v1.CanaryTestFailInstance{
Name: test.Name,
Url: fmt.Sprintf("%s%s", html.BugSearchUrl, encodedTestName),
PassRate: PassRate{
PassRate: v1.PassRate{
Percentage: test.PassPercentage,
Runs: test.Successes + test.Failures,
},
Expand All @@ -337,11 +282,11 @@ func canaryTestFailures(result map[string]util.SortedAggregateTestResult) []Cana
}

// job runs with failure groups
func failureGroupList(report util.TestReport) []FailureGroup {
func failureGroupList(report v12.TestReport) []v1.FailureGroup {

var failureGroups []FailureGroup
var failureGroups []v1.FailureGroup
for _, fg := range report.FailureGroups {
failureGroups = append(failureGroups, FailureGroup{
failureGroups = append(failureGroups, v1.FailureGroup{
Job: fg.Job,
Url: fg.Url,
TestFailures: fg.TestFailures,
Expand All @@ -350,7 +295,7 @@ func failureGroupList(report util.TestReport) []FailureGroup {
return failureGroups
}

func formatJSONReport(report, prevReport util.TestReport, endDay, jobTestCount int) map[string]interface{} {
func formatJSONReport(report, prevReport v12.TestReport, endDay, jobTestCount int) map[string]interface{} {
data := html.TestReports{
Current: report,
Prev: prevReport,
Expand All @@ -373,7 +318,7 @@ func formatJSONReport(report, prevReport util.TestReport, endDay, jobTestCount i
}

// PrintJSONReport prints json format of the reports
func PrintJSONReport(w http.ResponseWriter, req *http.Request, releaseReports map[string][]util.TestReport, endDay, jobTestCount int) {
func PrintJSONReport(w http.ResponseWriter, req *http.Request, releaseReports map[string][]v12.TestReport, endDay, jobTestCount int) {
reportObjects := make(map[string]interface{})
for _, reports := range releaseReports {
report := reports[0]
Expand Down
22 changes: 22 additions & 0 deletions pkg/apis/bugs/v1/types.go
View file
Open in desktop
@@ -0,0 +1,22 @@
package v1

import "time"

// Bug is used to represent bugs in some serialized content. It also appears to be used internally for data tracking
type Bug struct {
BugzillaBug `json:",inline"`
Url string `json:"url"`
FailureCount int `json:"failureCount,omitempty"`
FlakeCount int `json:"flakeCount,omitempty"`
}

// BugzillaBug matches the bugzilla API. We cannot change this and should consider writing a converter instead of having
// a serialization we don't own.
type BugzillaBug struct {
ID int64 `json:"id"`
Status string `json:"status"`
LastChangeTime time.Time `json:"last_change_time"`
Summary string `json:"summary"`
TargetRelease []string `json:"target_release"`
Component []string `json:"component"`
}