/
test_expectations_editor.py
167 lines (145 loc) · 8.95 KB
/
test_expectations_editor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A helper class for reading in and dealing with tests expectations
for layout tests.
"""
import itertools
import json
import logging
import re
from webkitpy.layout_tests.models.test_configuration import TestConfiguration, TestConfigurationConverter
from webkitpy.layout_tests.models import test_expectations
_log = logging.getLogger(__name__)
class BugManager(object):
"""A simple interface for managing bugs from TestExpectationsEditor."""
def close_bug(self, bug_ids, reference_bug_ids=None):
raise NotImplementedError("BugManager.close_bug")
def create_bug(self):
"""Should return a newly created bug id in the form of r"BUG[^\d].*"."""
raise NotImplementedError("BugManager.create_bug")
class TestExpectationsEditor(object):
"""
The editor assumes that the expectation data is error-free.
"""
def __init__(self, expectation_lines, bug_manager):
self._bug_manager = bug_manager
self._expectation_lines = expectation_lines
self._tests_with_directory_paths = set()
# FIXME: Unify this with TestExpectationsModel.
self._test_to_expectation_lines = {}
for expectation_line in expectation_lines:
for test in expectation_line.matching_tests:
if test == expectation_line.path:
self._test_to_expectation_lines.setdefault(test, []).append(expectation_line)
else:
self._tests_with_directory_paths.add(test)
def remove_expectation(self, test, test_config_set, remove_flakes=False):
"""Removes existing expectations for {test} in the of test configurations {test_config_set}.
If the test is flaky, the expectation is not removed, unless remove_flakes is True.
In this context, removing expectations does not imply that the test is passing -- we are merely removing
any information about this test from the expectations.
We do not remove the actual expectation lines here. Instead, we adjust TestExpectationLine.matching_configurations.
The serializer will figure out what to do:
* An empty matching_configurations set means that the this line matches nothing and will serialize as None.
* A matching_configurations set that can't be expressed as one line will be serialized as multiple lines.
Also, we do only adjust matching_configurations for lines that match tests exactly, because expectation lines with
better path matches are valid and always win.
For example, the expectation with the path "fast/events/shadow/" will
be ignored when removing expectations for the test "fast/event/shadow/awesome-crash.html", since we can just
add a new expectation line for "fast/event/shadow/awesome-crash.html" to influence expected results.
"""
expectation_lines = self._test_to_expectation_lines.get(test, [])
for expectation_line in expectation_lines:
if (not expectation_line.is_flaky() or remove_flakes) and expectation_line.matching_configurations & test_config_set:
expectation_line.matching_configurations = expectation_line.matching_configurations - test_config_set
if not expectation_line.matching_configurations:
self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers)
return
def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifiers=None):
"""Updates expectations for {test} in the set of test configuration {test_config_set} to the values of {expectation_set}.
If {parsed_bug_modifiers} is supplied, it is used for updated expectations. Otherwise, a new bug is created.
Here, we treat updating expectations to PASS as special: if possible, the corresponding lines are completely removed.
"""
# FIXME: Allow specifying modifiers (SLOW, SKIP, WONTFIX).
updated_expectations = []
expectation_lines = self._test_to_expectation_lines.get(test, [])
remaining_configurations = test_config_set.copy()
bug_ids = self._get_valid_bug_ids(parsed_bug_modifiers)
new_expectation_line_insertion_point = len(self._expectation_lines)
remove_expectations = expectation_set == set([test_expectations.PASS]) and test not in self._tests_with_directory_paths
for expectation_line in expectation_lines:
if expectation_line.matching_configurations == remaining_configurations:
# Tweak expectations on existing line.
if expectation_line.parsed_expectations == expectation_set:
return updated_expectations
self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
updated_expectations.append(expectation_line)
if remove_expectations:
expectation_line.matching_configurations = set()
else:
expectation_line.parsed_expectations = expectation_set
expectation_line.parsed_bug_modifiers = bug_ids
return updated_expectations
elif expectation_line.matching_configurations >= remaining_configurations:
# 1) Split up into two expectation lines:
# * one with old expectations (existing expectation_line)
# * one with new expectations (new expectation_line)
# 2) Finish looking, since there will be no more remaining configs to test for.
expectation_line.matching_configurations -= remaining_configurations
updated_expectations.append(expectation_line)
new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1
break
elif expectation_line.matching_configurations <= remaining_configurations:
# Remove existing expectation line.
self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
expectation_line.matching_configurations = set()
updated_expectations.append(expectation_line)
else:
intersection = expectation_line.matching_configurations & remaining_configurations
if intersection:
expectation_line.matching_configurations -= intersection
updated_expectations.append(expectation_line)
new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1
if not remove_expectations:
new_expectation_line = self._create_new_line(test, bug_ids, remaining_configurations, expectation_set)
updated_expectations.append(new_expectation_line)
self._expectation_lines.insert(new_expectation_line_insertion_point, new_expectation_line)
return updated_expectations
def _get_valid_bug_ids(self, suggested_bug_ids):
# FIXME: Flesh out creating a bug properly (title, etc.)
return suggested_bug_ids or [self._bug_manager.create_bug()]
def _create_new_line(self, name, bug_ids, config_set, expectation_set):
new_line = test_expectations.TestExpectationLine()
new_line.name = name
new_line.parsed_bug_modifiers = bug_ids
new_line.matching_configurations = config_set
new_line.parsed_expectations = expectation_set
# Ensure index integrity for multiple operations.
self._test_to_expectation_lines.setdefault(name, []).append(new_line)
return new_line