Skip to content

Commit 990fdd2

Browse files
committed
Bug 1874972 - sum test results for duration and use specific nested group names. r=ahal
Differential Revision: https://phabricator.services.mozilla.com/D199310
1 parent 5f93f33 commit 990fdd2

File tree

6 files changed

+65
-26
lines changed

6 files changed

+65
-26
lines changed

testing/mochitest/runtests.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ def __init__(self, logger, buffering=True, structured=True):
176176
self.structured = structured
177177
self.gecko_id = "GECKO"
178178
self.is_test_running = False
179+
self._manifest = None
179180

180181
# Even if buffering is enabled, we only want to buffer messages between
181182
# TEST-START/TEST-END. So it is off to begin, but will be enabled after
@@ -189,6 +190,9 @@ def __init__(self, logger, buffering=True, structured=True):
189190
# Message buffering
190191
self.buffered_messages = []
191192

193+
def setManifest(self, name):
194+
self._manifest = name
195+
192196
def validate(self, obj):
193197
"""Tests whether the given object is a valid structured message
194198
(only does a superficial validation)"""
@@ -257,6 +261,7 @@ def parse_line(self, line):
257261
self._fix_subtest_name(message)
258262
self._fix_test_name(message)
259263
self._fix_message_format(message)
264+
message["group"] = self._manifest
260265
messages.append(message)
261266

262267
return messages
@@ -3461,6 +3466,7 @@ def runTests(self, options):
34613466
for m in sorted(manifests):
34623467
self.log.group_start(name=m)
34633468
self.log.info("Running manifest: {}".format(m))
3469+
self.message_logger.setManifest(m)
34643470

34653471
args = list(self.args_by_manifest[m])[0]
34663472
self.extraArgs = []

testing/mozbase/mozlog/mozlog/formatters/errorsummary.py

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ def __init__(self):
1616
self.groups = defaultdict(
1717
lambda: {
1818
"status": None,
19+
"test_times": [],
1920
"start": None,
2021
"end": None,
2122
}
@@ -88,10 +89,7 @@ def suite_start(self, item):
8889
def suite_end(self, data):
8990
output = []
9091
for group, info in self.groups.items():
91-
if info["start"] is None or info["end"] is None:
92-
duration = None
93-
else:
94-
duration = info["end"] - info["start"]
92+
duration = sum(info["test_times"])
9593

9694
output.append(
9795
self._output(
@@ -107,12 +105,16 @@ def suite_end(self, data):
107105
return "".join(output)
108106

109107
def test_start(self, item):
110-
group = self.test_to_group.get(self._clean_test_name(item["test"]), None)
108+
group = item.get(
109+
"group", self.test_to_group.get(self._clean_test_name(item["test"]), None)
110+
)
111111
if group and self.groups[group]["start"] is None:
112112
self.groups[group]["start"] = item["time"]
113113

114114
def test_status(self, item):
115-
group = self.test_to_group.get(self._clean_test_name(item["test"]), None)
115+
group = item.get(
116+
"group", self.test_to_group.get(self._clean_test_name(item["test"]), None)
117+
)
116118
if group:
117119
self.groups[group]["status"] = self._get_group_result(group, item)
118120

@@ -127,10 +129,16 @@ def test_status(self, item):
127129
)
128130

129131
def test_end(self, item):
130-
group = self.test_to_group.get(self._clean_test_name(item["test"]), None)
132+
group = item.get(
133+
"group", self.test_to_group.get(self._clean_test_name(item["test"]), None)
134+
)
131135
if group:
132136
self.groups[group]["status"] = self._get_group_result(group, item)
133-
self.groups[group]["end"] = item["time"]
137+
if self.groups[group]["start"]:
138+
self.groups[group]["test_times"].append(
139+
item["time"] - self.groups[group]["start"]
140+
)
141+
self.groups[group]["start"] = None
134142

135143
if not self.dump_passing_tests and "expected" not in item:
136144
return

testing/mozbase/mozlog/mozlog/structuredlog.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
test - ID for the test
5353
path - Relative path to test (optional)
5454
subsuite - Name of the subsuite to which test belongs (optional)
55+
group - Name of the test group for the incoming test (optional)
5556
5657
test_end
5758
test - ID for the test
@@ -63,6 +64,7 @@
6364
known_intermittent - List of known intermittent statuses that should
6465
not fail a test. eg. ['FAIL', 'TIMEOUT']
6566
subsuite - Name of the subsuite to which test belongs (optional)
67+
group - Name of the test group for the incoming test (optional)
6668
6769
test_status
6870
test - ID for the test
@@ -74,6 +76,7 @@
7476
known_intermittent - List of known intermittent statuses that should
7577
not fail a test. eg. ['FAIL', 'TIMEOUT']
7678
subsuite - Name of the subsuite to which test belongs (optional)
79+
group - Name of the test group for the incoming test (optional)
7780
7881
process_output
7982
process - PID of the process
@@ -441,6 +444,7 @@ def suite_end(self, data):
441444
TestId("test"),
442445
Unicode("path", default=None, optional=True),
443446
Unicode("subsuite", default=None, optional=True),
447+
Unicode("group", default=None, optional=True),
444448
)
445449
def test_start(self, data):
446450
"""Log a test_start message
@@ -449,6 +453,8 @@ def test_start(self, data):
449453
:param path: Path to test relative to some base (typically the root of
450454
the source tree).
451455
:param subsuite: Optional name of the subsuite to which the test belongs.
456+
:param group: Optional name of the test group or manifest name (useful
457+
when running in paralle)
452458
"""
453459
if not self._state.suite_started:
454460
self.error(
@@ -472,6 +478,7 @@ def test_start(self, data):
472478
Dict(Any, "extra", default=None, optional=True),
473479
List(SubStatus, "known_intermittent", default=None, optional=True),
474480
Unicode("subsuite", default=None, optional=True),
481+
Unicode("group", default=None, optional=True),
475482
)
476483
def test_status(self, data):
477484
"""
@@ -487,6 +494,8 @@ def test_status(self, data):
487494
:param extra: Optional suite-specific data associated with the test result.
488495
:param known_intermittent: Optional list of string expected intermittent statuses
489496
:param subsuite: Optional name of the subsuite to which the test belongs.
497+
:param group: Optional name of the test group or manifest name (useful
498+
when running in paralle)
490499
"""
491500

492501
if data["expected"] == data["status"] or data["status"] == "SKIP":
@@ -511,6 +520,7 @@ def test_status(self, data):
511520
Dict(Any, "extra", default=None, optional=True),
512521
List(Status, "known_intermittent", default=None, optional=True),
513522
Unicode("subsuite", default=None, optional=True),
523+
Unicode("group", default=None, optional=True),
514524
)
515525
def test_end(self, data):
516526
"""
@@ -526,6 +536,8 @@ def test_end(self, data):
526536
:param stack: Optional stack trace encountered during test execution.
527537
:param extra: Optional suite-specific data associated with the test result.
528538
:param subsuite: Optional name of the subsuite to which the test belongs.
539+
:param group: Optional name of the test group or manifest name (useful
540+
when running in paralle)
529541
"""
530542

531543
if data["expected"] == data["status"] or data["status"] == "SKIP":

testing/mozbase/mozlog/tests/test_errorsummary.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
"""
3535
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
3636
{"test": "test_baz", "subtest": null, "group": "manifestA", "status": "PASS", "expected": "FAIL", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
37-
{"group": "manifestA", "status": "ERROR", "duration": 50, "action": "group_result", "line": 9}
37+
{"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9}
3838
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
3939
""".strip(),
4040
id="basic",
@@ -47,7 +47,7 @@
4747
],
4848
"""
4949
{"groups": ["manifest"], "action": "test_groups", "line": 0}
50-
{"group": "manifest", "status": null, "duration": null, "action": "group_result", "line": 2}
50+
{"group": "manifest", "status": null, "duration": 0, "action": "group_result", "line": 2}
5151
""".strip(),
5252
id="missing_test_end",
5353
),
@@ -87,7 +87,7 @@
8787
"""
8888
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
8989
{"test": "test_bar", "subtest": null, "group": "manifestA", "status": "CRASH", "expected": "OK", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 4}
90-
{"group": "manifestA", "status": "ERROR", "duration": 50, "action": "group_result", "line": 9}
90+
{"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9}
9191
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
9292
""".strip(),
9393
id="crash_and_group_status",
@@ -114,7 +114,7 @@
114114
],
115115
"""
116116
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
117-
{"group": "manifestA", "status": "OK", "duration": 59, "action": "group_result", "line": 10}
117+
{"group": "manifestA", "status": "OK", "duration": 29, "action": "group_result", "line": 10}
118118
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 10}
119119
""".strip(),
120120
id="fail_expected_fail",
@@ -146,7 +146,7 @@
146146
{"test": "test_baz", "subtest": "Test timed out", "group": "manifestA", "status": "FAIL", "expected": "PASS", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
147147
{"test": "test_baz", "subtest": "", "group": "manifestA", "status": "TIMEOUT", "expected": "PASS", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 9}
148148
{"test": "manifestA", "group": "manifestA", "signature": "signature", "stackwalk_stderr": null, "stackwalk_stdout": null, "action": "crash", "line": 10}
149-
{"group": "manifestA", "status": "ERROR", "duration": 79, "action": "group_result", "line": 12}
149+
{"group": "manifestA", "status": "ERROR", "duration": 49, "action": "group_result", "line": 12}
150150
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 12}
151151
""".strip(),
152152
id="timeout_and_crash",
@@ -172,7 +172,7 @@
172172
],
173173
"""
174174
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
175-
{"group": "manifestA", "status": "OK", "duration": 50, "action": "group_result", "line": 9}
175+
{"group": "manifestA", "status": "OK", "duration": 20, "action": "group_result", "line": 9}
176176
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
177177
""".strip(),
178178
id="crash_expected_crash",
@@ -200,7 +200,7 @@
200200
"""
201201
{"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
202202
{"test": "manifestA", "group": "manifestA", "signature": "", "stackwalk_stderr": null, "stackwalk_stdout": null, "action": "crash", "line": 9}
203-
{"group": "manifestA", "status": "ERROR", "duration": 50, "action": "group_result", "line": 10}
203+
{"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 10}
204204
{"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 10}
205205
""".strip(),
206206
id="assertion_crash_on_shutdown",

testing/mozbase/mozlog/tests/test_structured.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,7 @@ def test_arguments(self):
760760
self.logger.test_status,
761761
"test1",
762762
"subtest1",
763+
"group1",
763764
"PASS",
764765
"FAIL",
765766
"message",

testing/xpcshell/runxpcshelltests.py

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,16 @@
106106
_cleanup_encoding_re = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]")
107107

108108

109+
def get_full_group_name(test):
110+
group = test["manifest"]
111+
if "ancestor_manifest" in test:
112+
ancestor_manifest = normsep(test["ancestor_manifest"])
113+
# Only change the group id if ancestor is not the generated root manifest.
114+
if "/" in ancestor_manifest:
115+
group = "{}:{}".format(ancestor_manifest, group)
116+
return group
117+
118+
109119
def _cleanup_encoding_repl(m):
110120
c = m.group(0)
111121
return "\\\\" if c == "\\" else "\\x{0:02X}".format(ord(c))
@@ -757,14 +767,15 @@ def run_test(self):
757767

758768
name = self.test_object["id"]
759769
path = self.test_object["path"]
770+
group = get_full_group_name(self.test_object)
760771

761772
# Check for skipped tests
762773
if "disabled" in self.test_object:
763774
message = self.test_object["disabled"]
764775
if not message:
765776
message = "disabled from xpcshell manifest"
766-
self.log.test_start(name)
767-
self.log.test_end(name, "SKIP", message=message)
777+
self.log.test_start(name, group=group)
778+
self.log.test_end(name, "SKIP", message=message, group=group)
768779

769780
self.retry = False
770781
self.keep_going = True
@@ -851,7 +862,7 @@ def run_test(self):
851862
process_output = None
852863

853864
try:
854-
self.log.test_start(name)
865+
self.log.test_start(name, group=group)
855866
if self.verbose:
856867
self.logCommand(name, self.command, test_dir)
857868

@@ -949,20 +960,24 @@ def run_test(self):
949960
"CRASH",
950961
expected=expected,
951962
message="Test ended before setting up the crash reporter",
963+
group=group,
952964
)
953965
elif self.retry:
954966
self.log.test_end(
955967
name,
956968
status,
957969
expected=status,
958970
message="Test failed or timed out, will retry",
971+
group=group,
959972
)
960973
self.clean_temp_dirs(path)
961974
if self.verboseIfFails and not self.verbose:
962975
self.log_full_output()
963976
return
964977
else:
965-
self.log.test_end(name, status, expected=expected, message=message)
978+
self.log.test_end(
979+
name, status, expected=expected, message=message, group=group
980+
)
966981
self.log_full_output()
967982

968983
self.failCount += 1
@@ -980,7 +995,9 @@ def run_test(self):
980995
if self.usingTSan and return_code == TSAN_EXIT_CODE_WITH_RACES:
981996
self.log_full_output()
982997

983-
self.log.test_end(name, status, expected=expected, message=message)
998+
self.log.test_end(
999+
name, status, expected=expected, message=message, group=group
1000+
)
9841001
if self.verbose:
9851002
self.log_full_output()
9861003

@@ -2096,12 +2113,7 @@ def runTestList(
20962113

20972114
tests_by_manifest = defaultdict(list)
20982115
for test in self.alltests:
2099-
group = test["manifest"]
2100-
if "ancestor_manifest" in test:
2101-
ancestor_manifest = normsep(test["ancestor_manifest"])
2102-
# Only change the group id if ancestor is not the generated root manifest.
2103-
if "/" in ancestor_manifest:
2104-
group = "{}:{}".format(ancestor_manifest, group)
2116+
group = get_full_group_name(test)
21052117
tests_by_manifest[group].append(test["id"])
21062118

21072119
self.log.suite_start(tests_by_manifest, name="xpcshell")

0 commit comments

Comments
 (0)