/
TestConfiguration.java
454 lines (409 loc) · 18.8 KB
/
TestConfiguration.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis.test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.devtools.build.lib.analysis.OptionsDiffPredicate;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.CoreOptionConverters.LabelConverter;
import com.google.devtools.build.lib.analysis.config.Fragment;
import com.google.devtools.build.lib.analysis.config.FragmentOptions;
import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
import com.google.devtools.build.lib.analysis.config.RequiresOptions;
import com.google.devtools.build.lib.analysis.test.TestShardingStrategy.ShardingStrategyConverter;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.packages.TestTimeout;
import com.google.devtools.build.lib.util.RegexFilter;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDefinition;
import com.google.devtools.common.options.OptionDocumentationCategory;
import com.google.devtools.common.options.OptionEffectTag;
import com.google.devtools.common.options.OptionMetadataTag;
import com.google.devtools.common.options.OptionsParser;
import com.google.devtools.common.options.OptionsParsingException;
import com.google.devtools.common.options.TriState;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/** Test-related options. */
@RequiresOptions(options = {TestConfiguration.TestOptions.class})
public class TestConfiguration extends Fragment {
public static final OptionsDiffPredicate SHOULD_INVALIDATE_FOR_OPTION_DIFF =
(options, changedOption, oldValue, newValue) -> {
if (TestOptions.ALWAYS_INVALIDATE_WHEN_CHANGED.contains(changedOption)) {
// changes in --trim_test_configuration itself or related flags always prompt invalidation
return true;
}
if (!changedOption.getField().getDeclaringClass().equals(TestOptions.class)) {
// options outside of TestOptions always prompt invalidation
return true;
}
// other options in TestOptions require invalidation when --trim_test_configuration is off
return !options.get(TestOptions.class).trimTestConfiguration;
};
/** Command-line options. */
public static class TestOptions extends FragmentOptions {
private static final ImmutableSet<OptionDefinition> ALWAYS_INVALIDATE_WHEN_CHANGED =
ImmutableSet.of(
OptionsParser.getOptionDefinitionByName(TestOptions.class, "trim_test_configuration"),
OptionsParser.getOptionDefinitionByName(
TestOptions.class, "experimental_retain_test_configuration_across_testonly"));
@Option(
name = "test_timeout",
defaultValue = "-1",
converter = TestTimeout.TestTimeoutConverter.class,
documentationCategory = OptionDocumentationCategory.TESTING,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Override the default test timeout values for test timeouts (in secs). If a single "
+ "positive integer value is specified it will override all categories. If 4 "
+ "comma-separated integers are specified, they will override the timeouts for "
+ "short, moderate, long and eternal (in that order). In either form, a value of "
+ "-1 tells blaze to use its default timeouts for that category.")
public Map<TestTimeout, Duration> testTimeout;
@Option(
name = "test_filter",
allowMultiple = false,
defaultValue = "null",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Specifies a filter to forward to the test framework. Used to limit "
+ "the tests run. Note that this does not affect which targets are built."
)
public String testFilter;
@Option(
name = "test_runner_fail_fast",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Forwards fail fast option to the test runner. The test runner should stop execution"
+ " upon first failure.")
public boolean testRunnerFailFast;
@Option(
name = "cache_test_results",
defaultValue = "auto",
abbrev = 't', // it's useful to toggle this on/off quickly
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"If set to 'auto', Bazel reruns a test if and only if: "
+ "(1) Bazel detects changes in the test or its dependencies, "
+ "(2) the test is marked as external, "
+ "(3) multiple test runs were requested with --runs_per_test, or"
+ "(4) the test previously failed. "
+ "If set to 'yes', Bazel caches all test results except for tests marked as "
+ "external. If set to 'no', Bazel does not cache any test results."
)
public TriState cacheTestResults;
@Deprecated
@Option(
name = "test_result_expiration",
defaultValue = "-1", // No expiration by default.
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help = "This option is deprecated and has no effect.")
public int testResultExpiration;
@Option(
name = "trim_test_configuration",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
effectTags = {
OptionEffectTag.LOADING_AND_ANALYSIS,
OptionEffectTag.LOSES_INCREMENTAL_STATE,
},
help =
"When enabled, test-related options will be cleared below the top level of the build."
+ " When this flag is active, tests cannot be built as dependencies of non-test"
+ " rules, but changes to test-related options will not cause non-test rules to be"
+ " re-analyzed.")
public boolean trimTestConfiguration;
@Option(
name = "experimental_retain_test_configuration_across_testonly",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
effectTags = {
OptionEffectTag.LOADING_AND_ANALYSIS,
OptionEffectTag.LOSES_INCREMENTAL_STATE,
},
help =
"When enabled, --trim_test_configuration will not trim the test configuration for rules"
+ " marked testonly=1. This is meant to reduce action conflict issues when non-test"
+ " rules depend on cc_test rules. No effect if --trim_test_configuration is"
+ " false.")
public boolean experimentalRetainTestConfigurationAcrossTestonly;
@Option(
name = "test_arg",
allowMultiple = true,
defaultValue = "null",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Specifies additional options and arguments that should be passed to the test "
+ "executable. Can be used multiple times to specify several arguments. "
+ "If multiple tests are executed, each of them will receive identical arguments. "
+ "Used only by the 'bazel test' command.")
public List<String> testArguments;
@Option(
name = "test_sharding_strategy",
defaultValue = "explicit",
converter = ShardingStrategyConverter.class,
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Specify strategy for test sharding: "
+ "'explicit' to only use sharding if the 'shard_count' BUILD attribute is "
+ "present. 'disabled' to never use test sharding.")
public TestShardingStrategy testShardingStrategy;
@Option(
name = "runs_per_test",
allowMultiple = true,
defaultValue = "1",
converter = RunsPerTestConverter.class,
documentationCategory = OptionDocumentationCategory.TESTING,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Specifies number of times to run each test. If any of those attempts fail for any"
+ " reason, the whole test is considered failed. Normally the value specified is"
+ " just an integer. Example: --runs_per_test=3 will run all tests 3 times."
+ " Alternate syntax: regex_filter@runs_per_test. Where runs_per_test stands for"
+ " an integer value and regex_filter stands for a list of include and exclude"
+ " regular expression patterns (Also see --instrumentation_filter). Example:"
+ " --runs_per_test=//foo/.*,-//foo/bar/.*@3 runs all tests in //foo/ except those"
+ " under foo/bar three times. This option can be passed multiple times. The most"
+ " recently passed argument that matches takes precedence. If nothing matches,"
+ " the test is only run once.")
public List<PerLabelOptions> runsPerTest;
@Option(
name = "runs_per_test_detects_flakes",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"If true, any shard in which at least one run/attempt passes and at least one "
+ "run/attempt fails gets a FLAKY status.")
public boolean runsPerTestDetectsFlakes;
@Option(
name = "experimental_cancel_concurrent_tests",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.LOADING_AND_ANALYSIS},
help =
"If true, then Blaze will cancel concurrently running tests on the first successful "
+ "run. This is only useful in combination with --runs_per_test_detects_flakes.")
public boolean cancelConcurrentTests;
@Option(
name = "coverage_support",
converter = LabelConverter.class,
defaultValue = "@bazel_tools//tools/test:coverage_support",
documentationCategory = OptionDocumentationCategory.TOOLCHAIN,
effectTags = {
OptionEffectTag.CHANGES_INPUTS,
OptionEffectTag.AFFECTS_OUTPUTS,
OptionEffectTag.LOADING_AND_ANALYSIS
},
help =
"Location of support files that are required on the inputs of every test action "
+ "that collects code coverage. Defaults to '//tools/test:coverage_support'."
)
public Label coverageSupport;
@Option(
name = "coverage_report_generator",
converter = LabelConverter.class,
defaultValue = "@bazel_tools//tools/test:coverage_report_generator",
documentationCategory = OptionDocumentationCategory.TOOLCHAIN,
effectTags = {
OptionEffectTag.CHANGES_INPUTS,
OptionEffectTag.AFFECTS_OUTPUTS,
OptionEffectTag.LOADING_AND_ANALYSIS
},
help =
"Location of the binary that is used to generate coverage reports. This must "
+ "currently be a filegroup that contains a single file, the binary. Defaults to "
+ "'//tools/test:coverage_report_generator'."
)
public Label coverageReportGenerator;
@Option(
name = "experimental_fetch_all_coverage_outputs",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.LOADING_AND_ANALYSIS},
help =
"If true, then Bazel fetches the entire coverage data directory for each test during a "
+ "coverage run.")
public boolean fetchAllCoverageOutputs;
@Option(
name = "incompatible_exclusive_test_sandboxed",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE},
help =
"If true, exclusive tests will run with sandboxed strategy. Add 'local' tag to force "
+ "an exclusive test run locally")
public boolean incompatibleExclusiveTestSandboxed;
@Option(
name = "experimental_split_coverage_postprocessing",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
effectTags = {OptionEffectTag.EXECUTION},
help = "If true, then Bazel will run coverage postprocessing for test in a new spawn.")
public boolean splitCoveragePostProcessing;
@Option(
name = "zip_undeclared_test_outputs",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.TESTING,
effectTags = {OptionEffectTag.TEST_RUNNER},
help = "If true, undeclared test outputs will be archived in a zip file.")
public boolean zipUndeclaredTestOutputs;
@Override
public FragmentOptions getHost() {
// Options here are either:
// 1. Applicable only for the test actions, which are relevant only for the top-level targets
// before host or exec transitions can apply.
// 2. Supposed to be build-universal and thus non-transitionable anyways
// (e.g. trim_test_configuration)
// And thus the options should just be copied and not reset by the exec transition (as
// resetting them has incremental performance drawbacks when these options change).
return clone();
}
}
private final TestOptions options;
private final ImmutableMap<TestTimeout, Duration> testTimeout;
private final boolean shouldInclude;
public TestConfiguration(BuildOptions buildOptions) {
this.shouldInclude = buildOptions.contains(TestOptions.class);
if (shouldInclude) {
TestOptions options = buildOptions.get(TestOptions.class);
this.options = options;
this.testTimeout = ImmutableMap.copyOf(options.testTimeout);
} else {
this.options = null;
this.testTimeout = null;
}
}
@Override
public boolean shouldInclude() {
return shouldInclude;
}
/** Returns test timeout mapping as set by --test_timeout options. */
public ImmutableMap<TestTimeout, Duration> getTestTimeout() {
return testTimeout;
}
public String getTestFilter() {
return options.testFilter;
}
public boolean getTestRunnerFailFast() {
return options.testRunnerFailFast;
}
public TriState cacheTestResults() {
return options.cacheTestResults;
}
public List<String> getTestArguments() {
return options.testArguments;
}
public TestShardingStrategy testShardingStrategy() {
return options.testShardingStrategy;
}
public Label getCoverageSupport() {
return options.coverageSupport;
}
public Label getCoverageReportGenerator() {
return options.coverageReportGenerator;
}
/**
* @return number of times the given test should run. If the test doesn't match any of the
* filters, runs it once.
*/
public int getRunsPerTestForLabel(Label label) {
for (PerLabelOptions perLabelRuns : Lists.reverse(options.runsPerTest)) {
if (perLabelRuns.isIncluded(label)) {
return Integer.parseInt(Iterables.getOnlyElement(perLabelRuns.getOptions()));
}
}
return 1;
}
public boolean runsPerTestDetectsFlakes() {
return options.runsPerTestDetectsFlakes;
}
public boolean cancelConcurrentTests() {
return options.cancelConcurrentTests;
}
public boolean fetchAllCoverageOutputs() {
return options.fetchAllCoverageOutputs;
}
public boolean incompatibleExclusiveTestSandboxed() {
return options.incompatibleExclusiveTestSandboxed;
}
public boolean splitCoveragePostProcessing() {
return options.splitCoveragePostProcessing;
}
public boolean getZipUndeclaredTestOutputs() {
return options.zipUndeclaredTestOutputs;
}
/**
* Option converter that han handle two styles of value for "--runs_per_test":
*
* <ul>
* <li>--runs_per_test=NUMBER: Run each test NUMBER times.
* <li>--runs_per_test=test_regex@NUMBER: Run each test that matches test_regex NUMBER times.
* This form can be repeated with multiple regexes.
* </ul>
*/
public static class RunsPerTestConverter extends PerLabelOptions.PerLabelOptionsConverter {
@Override
public PerLabelOptions convert(String input) throws OptionsParsingException {
try {
return parseAsInteger(input);
} catch (NumberFormatException ignored) {
return parseAsRegex(input);
}
}
private PerLabelOptions parseAsInteger(String input)
throws NumberFormatException, OptionsParsingException {
int numericValue = Integer.parseInt(input);
if (numericValue <= 0) {
throw new OptionsParsingException("'" + input + "' should be >= 1");
} else {
RegexFilter catchAll =
new RegexFilter(Collections.singletonList(".*"), Collections.<String>emptyList());
return new PerLabelOptions(catchAll, Collections.singletonList(input));
}
}
private PerLabelOptions parseAsRegex(String input) throws OptionsParsingException {
PerLabelOptions testRegexps = super.convert(input);
if (testRegexps.getOptions().size() != 1) {
throw new OptionsParsingException("'" + input + "' has multiple runs for a single pattern");
}
String runsPerTest = Iterables.getOnlyElement(testRegexps.getOptions());
try {
int numericRunsPerTest = Integer.parseInt(runsPerTest);
if (numericRunsPerTest <= 0) {
throw new OptionsParsingException("'" + input + "' has a value < 1");
}
} catch (NumberFormatException e) {
throw new OptionsParsingException("'" + input + "' has a non-numeric value", e);
}
return testRegexps;
}
@Override
public String getTypeDescription() {
return "a positive integer or test_regex@runs. This flag may be passed more than once";
}
}
}