Skip to content

Commit

Permalink
Implement screenshot test for flutter web. (#45530)
Browse files Browse the repository at this point in the history
  • Loading branch information
chingjun committed Dec 6, 2019
1 parent 9233b53 commit c2eb068
Show file tree
Hide file tree
Showing 19 changed files with 1,093 additions and 131 deletions.
24 changes: 24 additions & 0 deletions .cirrus.yml
Expand Up @@ -153,7 +153,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-1-linux
Expand All @@ -162,7 +165,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-2-linux
Expand All @@ -171,7 +177,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-3-linux
Expand All @@ -180,7 +189,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-4-linux
Expand All @@ -189,7 +201,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-5-linux
Expand All @@ -198,7 +213,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-6-linux
Expand All @@ -207,7 +225,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: web_tests-7_last-linux # last Web shard must end with _last
Expand All @@ -216,7 +237,10 @@ task:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
script:
- ./dev/bots/download_goldctl.sh
- dart --enable-asserts ./dev/bots/test.dart

- name: build_tests-linux
Expand Down
1 change: 0 additions & 1 deletion dev/bots/test.dart
Expand Up @@ -68,7 +68,6 @@ const List<String> kWebTestFileBlacklist = <String>[
'test/widgets/selectable_text_test.dart',
'test/widgets/color_filter_test.dart',
'test/widgets/editable_text_cursor_test.dart',
'test/widgets/shadow_test.dart',
'test/widgets/raw_keyboard_listener_test.dart',
'test/widgets/editable_text_test.dart',
'test/widgets/widget_inspector_test.dart',
Expand Down
4 changes: 2 additions & 2 deletions packages/flutter/test/widgets/shadow_test.dart
Expand Up @@ -33,7 +33,7 @@ void main() {
matchesGoldenFile('shadow.BoxDecoration.enabled.png'),
);
debugDisableShadows = true;
}, skip: isBrowser);
});

testWidgets('Shadows on ShapeDecoration', (WidgetTester tester) async {
debugDisableShadows = false;
Expand Down Expand Up @@ -93,7 +93,7 @@ void main() {
matchesGoldenFile('shadow.PhysicalModel.enabled.png'),
);
debugDisableShadows = true;
}, skip: isBrowser);
});

testWidgets('Shadows with PhysicalShape', (WidgetTester tester) async {
debugDisableShadows = false;
Expand Down
21 changes: 13 additions & 8 deletions packages/flutter_goldens_client/lib/skia_client.dart
Expand Up @@ -19,6 +19,7 @@ import 'package:process/process.dart';
const String _kFlutterRootKey = 'FLUTTER_ROOT';
const String _kGoldctlKey = 'GOLDCTL';
const String _kServiceAccountKey = 'GOLD_SERVICE_ACCOUNT';
const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';

/// A client for uploading image tests and making baseline requests to the
/// Flutter Gold Dashboard.
Expand Down Expand Up @@ -408,14 +409,16 @@ class SkiaGoldClient {
/// Returns a JSON String with keys value pairs used to uniquely identify the
/// configuration that generated the given golden file.
///
/// Currently, the only key value pair being tracked is the platform the image
/// was rendered on.
/// Currently, the only key value pairs being tracked is the platform the
/// image was rendered on, and for web tests, the browser the image was
/// rendered on.
String _getKeysJSON() {
return json.encode(
<String, dynamic>{
'Platform' : platform.operatingSystem,
}
);
final Map<String, dynamic> keys = <String, dynamic>{
'Platform' : platform.operatingSystem,
};
if (platform.environment[_kTestBrowserKey] != null)
keys['Browser'] = platform.environment[_kTestBrowserKey];
return json.encode(keys);
}

/// Removes the file extension from the [fileName] to represent the test name
Expand Down Expand Up @@ -455,7 +458,7 @@ class SkiaGoldDigest {
return SkiaGoldDigest(
imageHash: json['digest'] as String,
paramSet: Map<String, dynamic>.from(json['paramset'] as Map<String, dynamic> ??
<String, String>{'Platform': 'none'}),
<String, List<String>>{'Platform': <String>[]}),
testName: json['test'] as String,
status: json['status'] as String,
);
Expand All @@ -477,6 +480,8 @@ class SkiaGoldDigest {
bool isValid(Platform platform, String name, String expectation) {
return imageHash == expectation
&& (paramSet['Platform'] as List<dynamic>).contains(platform.operatingSystem)
&& (platform.environment[_kTestBrowserKey] == null
|| paramSet['Browser'] == platform.environment[_kTestBrowserKey])
&& testName == name
&& status == 'positive';
}
Expand Down
15 changes: 15 additions & 0 deletions packages/flutter_test/lib/src/_goldens_io.dart
Expand Up @@ -6,7 +6,9 @@ import 'dart:async';
import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';
import 'dart:ui';

import 'package:flutter/widgets.dart' show Element;
import 'package:image/image.dart';
import 'package:path/path.dart' as path;
// ignore: deprecated_member_use
Expand Down Expand Up @@ -240,3 +242,16 @@ ComparisonResult compareLists(List<int> test, List<int> master) {
}
return ComparisonResult(passed: true);
}

/// An unsupported [WebGoldenComparator] that exists for API compatibility.
class DefaultWebGoldenComparator extends WebGoldenComparator {
@override
Future<bool> compare(Element element, Size size, Uri golden) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
}

@override
Future<void> update(Uri golden, Element element, Size size) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
}
}
69 changes: 65 additions & 4 deletions packages/flutter_test/lib/src/_goldens_web.dart
Expand Up @@ -2,11 +2,19 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:typed_data';
import 'dart:convert';
import 'dart:html' as html;
import 'dart:typed_data';
import 'dart:ui';

import 'goldens.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/widgets.dart';
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' as test_package show TestFailure;

/// An unsupported [GoldenFileComparator] that exists for API compatibility.
import 'goldens.dart';

/// An unsupported [GoldenFileComparator] that exists for API compatibility.
class LocalFileComparator extends GoldenFileComparator {
@override
Future<bool> compare(Uint8List imageBytes, Uri golden) {
Expand All @@ -19,10 +27,63 @@ class LocalFileComparator extends GoldenFileComparator {
}
}

/// Returns whether [test] and [master] are pixel by pixel identical.
/// Returns whether [test] and [master] are pixel by pixel identical.
///
/// This method is not supported on the web and throws an [UnsupportedError]
/// when called.
ComparisonResult compareLists(List<int> test, List<int> master) {
throw UnsupportedError('Golden testing is not supported on the web.');
}

/// The default [WebGoldenComparator] implementation for `flutter test`.
///
/// This comparator will send a request to the test server for golden comparison
/// which will then defer the comparison to [goldenFileComparator].
///
/// See also:
///
/// * [matchesGoldenFile], the function from [flutter_test] that invokes the
/// comparator.
class DefaultWebGoldenComparator extends WebGoldenComparator {
/// Creates a new [DefaultWebGoldenComparator] for the specified [testFile].
///
/// Golden file keys will be interpreted as file paths relative to the
/// directory in which [testFile] resides.
///
/// The [testFile] URL must represent a file.
DefaultWebGoldenComparator(this.testUri);

/// The test file currently being executed.
///
/// Golden file keys will be interpreted as file paths relative to the
/// directory in which this file resides.
Uri testUri;

@override
Future<bool> compare(Element element, Size size, Uri golden) async {
final String key = golden.toString();

final html.HttpRequest request = await html.HttpRequest.request(
'flutter_goldens',
method: 'POST',
sendData: json.encode(<String, Object>{
'testUri': testUri.toString(),
'key': key.toString(),
'width': size.width.round(),
'height': size.height.round(),
}),
);
final String response = request.response as String;
if (response == 'true') {
return true;
} else {
throw test_package.TestFailure(response);
}
}

@override
Future<void> update(Uri golden, Element element, Size size) async {
// Update is handled on the server side, just use the same logic here
await compare(element, size, golden);
}
}
95 changes: 95 additions & 0 deletions packages/flutter_test/lib/src/_matchers_io.dart
@@ -0,0 +1,95 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:typed_data';
import 'dart:ui' as ui;

import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/services.dart';
import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf;

import 'binding.dart';
import 'finders.dart';
import 'goldens.dart';

/// Render the closest [RepaintBoundary] of the [element] into an image.
///
/// See also:
///
/// * [OffsetLayer.toImage] which is the actual method being called.
Future<ui.Image> captureImage(Element element) {
RenderObject renderObject = element.renderObject;
while (!renderObject.isRepaintBoundary) {
renderObject = renderObject.parent as RenderObject;
assert(renderObject != null);
}
assert(!renderObject.debugNeedsPaint);
final OffsetLayer layer = renderObject.debugLayer as OffsetLayer;
return layer.toImage(renderObject.paintBounds);
}

/// The matcher created by [matchesGoldenFile]. This class is enabled when the
/// test is running on a VM using conditional import.
class MatchesGoldenFile extends AsyncMatcher {
/// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
const MatchesGoldenFile(this.key, this.version);

/// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path);

/// The [key] to the golden image.
final Uri key;

/// The [version] of the golden image.
final int version;

@override
Future<String> matchAsync(dynamic item) async {
Future<ui.Image> imageFuture;
if (item is Future<ui.Image>) {
imageFuture = item;
} else if (item is ui.Image) {
imageFuture = Future<ui.Image>.value(item);
} else {
final Finder finder = item as Finder;
final Iterable<Element> elements = finder.evaluate();
if (elements.isEmpty) {
return 'could not be rendered because no widget was found';
} else if (elements.length > 1) {
return 'matched too many widgets';
}
imageFuture = captureImage(elements.single);
}

final Uri testNameUri = goldenFileComparator.getTestUri(key, version);

final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding;
return binding.runAsync<String>(() async {
final ui.Image image = await imageFuture;
final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png);
if (bytes == null)
return 'could not encode screenshot.';
if (autoUpdateGoldenFiles) {
await goldenFileComparator.update(testNameUri, bytes.buffer.asUint8List());
return null;
}
try {
final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), testNameUri);
return success ? null : 'does not match';
} on TestFailure catch (ex) {
return ex.message;
}
}, additionalTime: const Duration(minutes: 1));
}

@override
Description describe(Description description) {
final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
return description.add('one widget whose rasterized image matches golden image "$testNameUri"');
}
}

0 comments on commit c2eb068

Please sign in to comment.