Skip to content
This repository has been archived by the owner on Dec 15, 2018. It is now read-only.

Commit

Permalink
Add flakiest tests to build report
Browse files Browse the repository at this point in the history 
Summary:
Flaky tests are tests that failed on the first pass and then
succeeded on retry. The report includes the total number of times each
test was flaky, and the percentage of the time that it was flaky among
builds in which the test eventually passed. (So if a new flaky test
was added near the end of the report period, it might have a low count
but a high percentage.)

Test Plan: manually ran on changes server

Reviewers: rbarton

Subscribers: kylec, changesbot, wwu

Projects: #changes, #server_tests_test_framework

Differential Revision: https://tails.corp.dropbox.com/D82541
  • Loading branch information
@aroravishal
aroravishal committed Mar 16, 2015
1 parent 3ce5eaa commit fd85f7e
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 21 deletions.
80 changes: 65 additions & 15 deletions changes/reports/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,16 @@

from collections import defaultdict
from datetime import datetime, timedelta
from hashlib import sha1
from sqlalchemy.sql import func

from changes.config import db
from changes.constants import Status, Result
from changes.models import Build, FailureReason, TestCase, Source
from changes.models import Build, FailureReason, TestCase, Source, Job
from changes.utils.http import build_uri


MAX_FLAKY_TESTS = 5
MAX_SLOW_TESTS = 10
SLOW_TEST_THRESHOLD = 3000 # ms

ONE_DAY = 60 * 60 * 24
Expand Down Expand Up @@ -102,6 +103,7 @@ def generate(self, days=7):
},
})

flaky_tests = self.get_flaky_tests(start_period, end_period)
slow_tests = self.get_slow_tests(start_period, end_period)

title = 'Build Report ({0} through {1})'.format(
Expand All @@ -117,6 +119,7 @@ def generate(self, days=7):
'failure_stats': failure_stats,
'project_stats': project_stats,
'tests': {
'flaky_list': flaky_tests,
'slow_list': slow_tests,
},
}
Expand Down Expand Up @@ -226,7 +229,7 @@ def get_slow_tests(self, start_period, end_period):
slow_tests.extend(self.get_slow_tests_for_project(
project, start_period, end_period))
slow_tests.sort(key=lambda x: x['duration_raw'], reverse=True)
return slow_tests[:10]
return slow_tests[:MAX_SLOW_TESTS]

def get_slow_tests_for_project(self, project, start_period, end_period):
latest_build = Build.query.filter(
Expand All @@ -246,27 +249,74 @@ def get_slow_tests_for_project(self, project, start_period, end_period):
if not job_list:
return []

queryset = db.session.query(
TestCase.name, TestCase.duration,
).filter(
queryset = TestCase.query.filter(
TestCase.job_id.in_(j.id for j in job_list),
TestCase.result == Result.passed,
TestCase.date_created > start_period,
TestCase.date_created <= end_period,
).group_by(
TestCase.name, TestCase.duration,
).order_by(TestCase.duration.desc())
).order_by(
TestCase.duration.desc()
).limit(MAX_SLOW_TESTS)

slow_list = []
for name, duration in queryset[:10]:
for test in queryset:
slow_list.append({
'project': project,
'name': name,
'package': '', # TODO
'duration': '%.2f s' % (duration / 1000.0,),
'duration_raw': duration,
'name': test.short_name,
'package': test.package,
'duration': '%.2f s' % (test.duration / 1000.0,),
'duration_raw': test.duration,
'link': build_uri('/projects/{0}/tests/{1}/'.format(
project.slug, sha1(name).hexdigest())),
project.slug, test.name_sha)),
})

return slow_list

def get_flaky_tests(self, start_period, end_period):
test_queryset = TestCase.query.join(
Job, Job.id == TestCase.job_id,
).join(
Build, Build.id == Job.build_id,
).filter(
Build.project_id.in_(p.id for p in self.projects),
TestCase.result == Result.passed,
Build.date_created >= start_period,
Build.date_created < end_period,
).join(
Source, Source.id == Build.source_id,
).filter(
Source.patch_id == None, # NOQA
)

flaky_test_queryset = test_queryset.with_entities(
TestCase.name,
func.sum(TestCase.reruns).label('reruns'),
func.avg(TestCase.reruns).label('rerun_avg'),
).group_by(
TestCase.name
).order_by(
func.sum(TestCase.reruns).desc()
).limit(MAX_FLAKY_TESTS)

flaky_list = []
for name, reruns, rerun_avg in flaky_test_queryset:
if reruns == 0:
continue

rerun = test_queryset.filter(
TestCase.name == name,
TestCase.reruns > 0,
).first()

flaky_list.append({
'name': rerun.short_name,
'package': rerun.package,
'reruns': '%d (%.1f%%)' % (reruns, 100 * rerun_avg),
'link': build_uri('/projects/{0}/builds/{1}/jobs/{2}/logs/{3}/'.format(
rerun.project.slug,
rerun.job.build.id.hex,
rerun.job.id.hex,
rerun.step.logsources[0].id.hex)),
})

return flaky_list
62 changes: 56 additions & 6 deletions templates/email/build_report.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
}
td, th {
font-size: 0.95em;
padding: 5px 8px;
line-height: 24px;
padding: 10px 8px;
line-height: 14px;
vertical-align: middle;
}
th {
Expand All @@ -45,6 +45,7 @@
thead th {
color: #6e5baa;
font-weight: 500;
vertical-align: bottom;
}

tbody th a {
Expand Down Expand Up @@ -107,12 +108,14 @@
color: #999;
}

tbody td.duration {
width: 30px;
table .value {
text-align: right;
white-space: nowrap;
}

tbody td.value {
vertical-align: top;
font-size: 12px;
white-space: nowrap;
}

tr.odd th {
Expand Down Expand Up @@ -304,13 +307,60 @@ <h2>Failure Breakdown</h2>
</table>
</div>

{% if tests.flaky_list %}
<div id="flakiest-tests">
<div class="subheader">
<h2>Flakiest Tests</h2>
</div>

<p>
<small>
A flaky run for a test is one in which the test failed initially,
and then passed when re-run.
</small>
</p>

<table>
<thead>
<tr>
<th>Test</th>
<th class="value">
Flaky runs<br>(% of passing runs)
</th>
</tr>
</thead>
<tbody>
{% for test in tests.flaky_list %}
<tr class="{{ loop.cycle('odd', 'even') }}">
<th>
<a href="{{ test.link }}" class="truncate">{{ test.name }}</a><br>
{% if test.package %}
<small class="truncate">{{ test.package }}</small>
{% endif %}
</th>
<td class="value">{{ test.reruns }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endif %}

{% if tests.slow_list %}
<div id="slowest-tests">
<div class="subheader">
<h2>Slowest Tests</h2>
</div>

<table>
<thead>
<tr>
<th>Test</th>
<th class="value">
Duration in<br>latest green build
</th>
</tr>
</thead>
<tbody>
{% for test in tests.slow_list %}
<tr class="{{ loop.cycle('odd', 'even') }}">
Expand All @@ -320,7 +370,7 @@ <h2>Slowest Tests</h2>
<small class="truncate">{{ test.package }}</small>
{% endif %}
</th>
<td class="duration">{{ test.duration }}</td>
<td class="value">{{ test.duration }}</td>
</tr>
{% endfor %}
</tbody>
Expand Down

0 comments on commit fd85f7e

Please sign in to comment.