Skip to content
Permalink
Browse files

Auto merge of #24981 - servo:wpt-unexpected, r=jdm

Improve diagnostics for WPT failures

* Include the full output (including stdout/stderr) in the intermittent-filtered log
* Print the intermittent-filtered log at the end of the main log (which is one less click to reach from Taskcluster’s task view, compared to other task artifacts)
* <del>Fail with a specific message when a reftest screenshot is entirely white</del> (This caused over a hundred unexpected results. A few of them in reftests that use `about:blank` as a reference.)
* For failing reftests, add a message if the whole screenshot is a solid color, to help recognize instances of #24726

  ```
    ▶ FAIL [expected PASS] /css/CSS2/box-display/root-box-003.xht
    │   → /css/CSS2/box-display/root-box-003.xht 54a9df64f1476dd12020019d7cf22ac34d727bc0
    │   → /css/CSS2/box-display/root-box-003-ref.xht 636eb693bc214b6e1c64e6566c48e69e6777b946
    └   → Screenshot is solid color 0xFFFFFF for /css/CSS2/box-display/root-box-003.xht
  ```
  (The last line is new.)
  • Loading branch information
bors-servo committed Dec 3, 2019
2 parents 87c1019 + d0785e0 commit 64b7a31cd71e3862b9ff9a4bec1eb6f47a747f55
@@ -201,6 +201,7 @@ pub struct IOCompositor<Window: WindowMethods + ?Sized> {
cursor_pos: DevicePoint,

output_file: Option<String>,
fail_if_blank: bool,

is_running_problem_test: bool,

@@ -278,6 +279,7 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
window: Rc<Window>,
state: InitialCompositorState,
output_file: Option<String>,
fail_if_blank: bool,
is_running_problem_test: bool,
exit_after_load: bool,
convert_mouse_to_touch: bool,
@@ -320,6 +322,7 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
cursor: Cursor::None,
cursor_pos: DevicePoint::new(0.0, 0.0),
output_file,
fail_if_blank,
is_running_problem_test,
exit_after_load,
convert_mouse_to_touch,
@@ -330,6 +333,7 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
window: Rc<Window>,
state: InitialCompositorState,
output_file: Option<String>,
fail_if_blank: bool,
is_running_problem_test: bool,
exit_after_load: bool,
convert_mouse_to_touch: bool,
@@ -338,6 +342,7 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
window,
state,
output_file,
fail_if_blank,
is_running_problem_test,
exit_after_load,
convert_mouse_to_touch,
@@ -1387,6 +1392,10 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
FramebufferUintLength::new(width),
FramebufferUintLength::new(height),
);
if self.fail_if_blank && crate::image_is_entirely_white(&img) {
eprintln!("Would screenshot an entirely white PNG with --fail-if-blank");
std::process::exit(1);
}
let dynamic_image = DynamicImage::ImageRgb8(img);
if let Err(e) = dynamic_image.write_to(&mut file, ImageFormat::PNG)
{
@@ -36,3 +36,27 @@ pub struct CompositionPipeline {
pub script_chan: IpcSender<ConstellationControlMsg>,
pub layout_chan: IpcSender<LayoutControlMsg>,
}

fn image_is_entirely_white(img: &image::RgbImage) -> bool {
let buffer: &[u8] = img;
// Safety: transmuting an aligned `[u8; size_of::<usize>()]` to `usize` is safe
#[allow(unsafe_code)]
let (before, aligned, after) = unsafe { buffer.align_to::<usize>() };
for &byte in before {
if byte != 0xFF_u8 {
return false;
}
}
for &chunk in aligned {
const FF: usize = !0;
if chunk != FF {
return false;
}
}
for &byte in after {
if byte != 0xFF_u8 {
return false;
}
}
return true;
}
@@ -61,6 +61,7 @@ pub struct Opts {
pub user_stylesheets: Vec<(Vec<u8>, ServoUrl)>,

pub output_file: Option<String>,
pub fail_if_blank: bool,

/// Replace unpaired surrogates in DOM strings with U+FFFD.
/// See <https://github.com/servo/servo/issues/6564>
@@ -527,6 +528,7 @@ pub fn default_opts() -> Opts {
userscripts: None,
user_stylesheets: Vec::new(),
output_file: None,
fail_if_blank: false,
replace_surrogates: false,
gc_profile: false,
load_webfonts_synchronously: false,
@@ -581,6 +583,11 @@ pub fn from_cmdline_args(mut opts: Options, args: &[String]) -> ArgumentParsingR
opts.optflag("c", "cpu", "CPU painting");
opts.optflag("g", "gpu", "GPU painting");
opts.optopt("o", "output", "Output file", "output.png");
opts.optflag(
"",
"fail-if-blank",
"Exit with an error code if the PNG file for -o/--output would be entirely white pixels",
);
opts.optopt("s", "size", "Size of tiles", "512");
opts.optflagopt(
"p",
@@ -950,6 +957,7 @@ pub fn from_cmdline_args(mut opts: Options, args: &[String]) -> ArgumentParsingR
userscripts: opt_match.opt_default("userscripts", ""),
user_stylesheets: user_stylesheets,
output_file: opt_match.opt_str("o"),
fail_if_blank: opt_match.opt_present("fail-if-blank"),
replace_surrogates: debug_options.replace_surrogates,
gc_profile: debug_options.gc_profile,
load_webfonts_synchronously: debug_options.load_webfonts_synchronously,
@@ -554,6 +554,7 @@ where
webxr_main_thread,
},
opts.output_file.clone(),
opts.fail_if_blank,
opts.is_running_problem_test,
opts.exit_after_load,
opts.convert_mouse_to_touch,
@@ -748,12 +748,12 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
| cat
time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
--headless --log-raw test-wdspec.log \
--log-errorsummary wdspec-errorsummary.log \
--log-servojson wdspec-jsonsummary.log \
--always-succeed \
webdriver \
| cat
./mach filter-intermittents \
wdspec-errorsummary.log \
wdspec-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wdspec-errorsummary.log \
--tracker-api default \
@@ -768,11 +768,11 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
--total-chunks "$TOTAL_CHUNKS" \
--this-chunk "$THIS_CHUNK" \
--log-raw test-wpt.log \
--log-errorsummary wpt-errorsummary.log \
--log-servojson wpt-jsonsummary.log \
--always-succeed \
| cat
./mach filter-intermittents \
wpt-errorsummary.log \
wpt-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wpt-errorsummary.log \
--tracker-api default \
@@ -68,6 +68,7 @@


def create_parser_wpt():
import mozlog.commandline
parser = wptcommandline.create_parser()
parser.add_argument('--release', default=False, action="store_true",
help="Run with a release build of servo")
@@ -77,6 +78,8 @@ def create_parser_wpt():
help="Pass preferences to servo")
parser.add_argument('--layout-2020', default=False, action="store_true",
help="Use expected results for the 2020 layout engine")
parser.add_argument('--log-servojson', action="append", type=mozlog.commandline.log_file,
help="Servo's JSON logger of unexpected results")
parser.add_argument('--always-succeed', default=False, action="store_true",
help="Always yield exit code of zero")
return parser
@@ -511,7 +514,7 @@ def update_wpt(self, **kwargs):
description='Given a WPT error summary file, filter out intermittents and other cruft.',
category='testing')
@CommandArgument('summary',
help="Error summary log to take un")
help="Error summary log to take in")
@CommandArgument('--log-filteredsummary', default=None,
help='Print filtered log to file')
@CommandArgument('--log-intermittents', default=None,
@@ -529,10 +532,7 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents,
encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '')
failures = []
with open(summary, "r") as file:
for line in file:
line_json = json.loads(line)
if 'status' in line_json:
failures += [line_json]
failures = [json.loads(line) for line in file]
actual_failures = []
intermittents = []
for failure in failures:
@@ -546,10 +546,7 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents,
request = urllib.request.Request("%s/query.py?name=%s" % (tracker_api, query))
search = urllib.request.urlopen(request)
data = json.load(search)
if len(data) == 0:
actual_failures += [failure]
else:
intermittents += [failure]
is_intermittent = len(data) > 0
else:
qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
# we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
@@ -559,28 +556,30 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents,
request.add_header("Authorization", "Basic %s" % encoded_auth)
search = urllib.request.urlopen(request)
data = json.load(search)
if data['total_count'] == 0:
actual_failures += [failure]
else:
intermittents += [failure]
is_intermittent = data['total_count'] > 0

if is_intermittent:
intermittents.append(failure["output"])
else:
actual_failures.append(failure["output"])

def format(outputs, description, file=sys.stdout):
print(len(outputs), description + ":\n", file=file)
file.write('\n'.join(outputs).encode("utf-8"))

if log_intermittents:
with open(log_intermittents, "w") as intermittents_file:
for intermittent in intermittents:
json.dump(intermittent, intermittents_file, indent=4)
print("\n", end='', file=intermittents_file)
with open(log_intermittents, "wb") as file:
format(intermittents, "known-intermittent unexpected results", file)

output = open(log_filteredsummary, "w") if log_filteredsummary else sys.stdout
for failure in actual_failures:
json.dump(failure, output, indent=4)
print("\n", end='', file=output)
description = "unexpected results that are NOT known-intermittents"
if log_filteredsummary:
with open(log_filteredsummary, "wb") as file:
format(actual_failures, description, file)

if output is not sys.stdout:
output.close()
if actual_failures:
format(actual_failures, description)

if len(actual_failures) == 0:
return 0
return 1
return bool(actual_failures)

@Command('test-android-startup',
description='Extremely minimal testing of Servo for Android',
@@ -4,6 +4,7 @@

from mozlog.formatters import base
import collections
import json
import os
import sys
import subprocess
@@ -14,7 +15,7 @@
DEFAULT_CLEAR_EOL_CODE = u"\x1b[K"


class GroupingFormatter(base.BaseFormatter):
class ServoFormatter(base.BaseFormatter):
"""Formatter designed to produce unexpected test results grouped
together in a readable format."""
def __init__(self):
@@ -77,7 +78,7 @@ def text_to_erase_display(self):
return ((self.move_up + self.clear_eol) *
self.current_display.count('\n'))

def generate_output(self, text=None, new_display=None):
def generate_output(self, text=None, new_display=None, unexpected_in_test=None):
if not self.interactive:
return text

@@ -230,7 +231,8 @@ def test_end(self, data):
subtest_failures)
self.test_failure_text += output

return self.generate_output(text=output, new_display=new_display)
return self.generate_output(text=output, new_display=new_display,
unexpected_in_test=test_name)

def test_status(self, data):
if "expected" in data:
@@ -289,3 +291,16 @@ def log(self, data):

if data['level'] in ('CRITICAL', 'ERROR'):
return self.generate_output(text=data['message'] + "\n")


class ServoJsonFormatter(ServoFormatter):
def suite_start(self, data):
ServoFormatter.suite_start(self, data)
# Don't forward the return value

def generate_output(self, text=None, new_display=None, unexpected_in_test=None):
if unexpected_in_test:
return "%s\n" % json.dumps({"test": unexpected_in_test, "output": text})

def log(self, _):
return
@@ -1,2 +1,2 @@
[root-box-002.xht]
expected: FAIL
expected: CRASH
@@ -0,0 +1,2 @@
[root-box-003.xht]
expected: CRASH
"support"
],
"tools/wptrunner/wptrunner/executors/executorservo.py": [
"365a1a08653b0175fc4df4050eb9d9aaf8b1d34c",
"6fd45d20c8ae5494ddf2046bac2f254ffbb1ab63",
"support"
],
"tools/wptrunner/wptrunner/executors/executorservodriver.py": [
@@ -34,7 +34,9 @@ def run_tests(**kwargs):
set_defaults(kwargs)

mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
(grouping_formatter.ServoFormatter, "Servo’s grouping output formatter")
mozlog.commandline.log_formatters["servojson"] = \
(grouping_formatter.ServoJsonFormatter, "Servo's JSON logger of unexpected results")

use_mach_logging = False
if len(kwargs["test_list"]) == 1:
@@ -198,7 +198,9 @@ def screenshot(self, test, viewport_size, dpi):
debug_args, command = browser_command(
self.binary,
[
"--hard-fail", "--exit",
"--exit",
"--hard-fail",
"--fail-if-blank",
"-u", "Servo/wptrunner",
"-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
"--output=%s" % output_path, full_url

0 comments on commit 64b7a31

Please sign in to comment.
You can’t perform that action at this time.