From e6e6a02106c6ee1ff5d3f71e0e0c378f80759728 Mon Sep 17 00:00:00 2001 From: alanwutang11 <50225138+alanwutang11@users.noreply.github.com> Date: Tue, 6 Dec 2022 07:21:33 -0800 Subject: [PATCH] implement targetWidth and targetHeight (#38028) * implement targetWidth and targetHeight * blank lines and typo * addressed comments * add warning to tests --- .../lib/src/engine/canvaskit/image.dart | 99 ++++++++++++++-- .../engine/canvaskit/image_wasm_codecs.dart | 35 +++++- .../engine/canvaskit/image_web_codecs.dart | 11 -- .../lib/src/engine/safe_browser_api.dart | 4 +- .../test/canvaskit/image_golden_test.dart | 106 +++++++++++++++++- 5 files changed, 229 insertions(+), 26 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index ad570e4110326..d9545a2e1bdeb 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -11,29 +11,29 @@ import '../dom.dart'; import '../html_image_codec.dart'; import '../safe_browser_api.dart'; import '../util.dart'; +import 'canvas.dart'; import 'canvaskit_api.dart'; import 'image_wasm_codecs.dart'; import 'image_web_codecs.dart'; +import 'painting.dart'; +import 'picture.dart'; +import 'picture_recorder.dart'; import 'skia_object_cache.dart'; /// Instantiates a [ui.Codec] backed by an `SkAnimatedImage` from Skia. -// TODO(yjbanov): Implement targetWidth and targetHeight support. -// https://github.com/flutter/flutter/issues/34075 FutureOr skiaInstantiateImageCodec(Uint8List list, [int? targetWidth, int? targetHeight]) { - if (browserSupportsImageDecoder) { + // If we have either a target width or target height, use canvaskit to decode. + if (browserSupportsImageDecoder && (targetWidth == null && targetHeight == null)) { return CkBrowserImageDecoder.create( data: list, debugSource: 'encoded image bytes', - targetWidth: targetWidth, - targetHeight: targetHeight, ); } else { - return CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes'); + return CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes', targetWidth: targetWidth, targetHeight: targetHeight); } } -// TODO(yjbanov): add support for targetWidth/targetHeight (https://github.com/flutter/flutter/issues/34075) void skiaDecodeImageFromPixels( Uint8List pixels, int width, @@ -45,6 +45,13 @@ void skiaDecodeImageFromPixels( int? targetHeight, bool allowUpscaling = true, }) { + if (targetWidth != null) { + assert(allowUpscaling || targetWidth <= width); + } + if (targetHeight != null) { + assert(allowUpscaling || targetHeight <= height); + } + // Run in a timer to avoid janking the current frame by moving the decoding // work outside the frame event. Timer.run(() { @@ -65,10 +72,88 @@ void skiaDecodeImageFromPixels( return; } + if (targetWidth != null || targetHeight != null) { + if (!validUpscale(allowUpscaling, targetWidth, targetHeight, width, height)) { + domWindow.console.warn('Cannot apply targetWidth/targetHeight when allowUpscaling is false.'); + } else { + return callback(scaleImage(skImage, targetWidth, targetHeight)); + } + } return callback(CkImage(skImage)); }); } +// An invalid upscale happens when allowUpscaling is false AND either the given +// targetWidth is larger than the originalWidth OR the targetHeight is larger than originalHeight. +bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, int originalWidth, int originalHeight) { + if (allowUpscaling) { + return true; + } + final bool targetWidthFits; + final bool targetHeightFits; + if (targetWidth != null) { + targetWidthFits = targetWidth <= originalWidth; + } else { + targetWidthFits = true; + } + + if (targetHeight != null) { + targetHeightFits = targetHeight <= originalHeight; + } else { + targetHeightFits = true; + } + return targetWidthFits && targetHeightFits; +} + +/// Creates a scaled [CkImage] from an [SkImage] by drawing the [SkImage] to a canvas. +/// +/// This function will only be called if either a targetWidth or targetHeight is not null +/// +/// If only one of targetWidth or targetHeight are specified, the other +/// dimension will be scaled according to the aspect ratio of the supplied +/// dimension. +/// +/// If either targetWidth or targetHeight is less than or equal to zero, it +/// will be treated as if it is null. +CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) { + assert(targetWidth != null || targetHeight != null); + if (targetWidth != null && targetWidth <= 0) { + targetWidth = null; + } + if (targetHeight != null && targetHeight <= 0) { + targetHeight = null; + } + if (targetWidth == null && targetHeight != null) { + targetWidth = (targetHeight * (image.width() / image.height())).round(); + targetHeight = targetHeight; + } else if (targetHeight == null && targetWidth != null) { + targetWidth = targetWidth; + targetHeight = targetWidth ~/ (image.width() / image.height()); + } + + assert(targetWidth != null); + assert(targetHeight != null); + + final CkPictureRecorder recorder = CkPictureRecorder(); + final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); + + canvas.drawImageRect( + CkImage(image), + ui.Rect.fromLTWH(0, 0, image.width(), image.height()), + ui.Rect.fromLTWH(0, 0, targetWidth!.toDouble(), targetHeight!.toDouble()), + CkPaint() + ); + + final CkPicture picture = recorder.endRecording(); + final ui.Image finalImage = picture.toImageSync( + targetWidth, + targetHeight + ); + + final CkImage ckImage = finalImage as CkImage; + return ckImage; +} + /// Thrown when the web engine fails to decode an image, either due to a /// network issue, corrupted image contents, or missing codec. class ImageCodecException implements Exception { diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart index e7772d7a10cb8..a83fc2453b28b 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_wasm_codecs.dart @@ -14,6 +14,7 @@ import 'dart:typed_data'; import 'package:ui/ui.dart' as ui; +import '../util.dart'; import 'canvaskit_api.dart'; import 'image.dart'; import 'skia_object_cache.dart'; @@ -24,7 +25,7 @@ import 'skia_object_cache.dart'; class CkAnimatedImage extends ManagedSkiaObject implements ui.Codec { /// Decodes an image from a list of encoded bytes. - CkAnimatedImage.decodeFromBytes(this._bytes, this.src); + CkAnimatedImage.decodeFromBytes(this._bytes, this.src, {this.targetWidth, this.targetHeight}); final String src; final Uint8List _bytes; @@ -34,9 +35,12 @@ class CkAnimatedImage extends ManagedSkiaObject /// Current frame index. int _currentFrameIndex = 0; + final int? targetWidth; + final int? targetHeight; + @override SkAnimatedImage createDefault() { - final SkAnimatedImage? animatedImage = + SkAnimatedImage? animatedImage = canvasKit.MakeAnimatedImageFromEncoded(_bytes); if (animatedImage == null) { throw ImageCodecException( @@ -45,6 +49,20 @@ class CkAnimatedImage extends ManagedSkiaObject ); } + if (targetWidth != null || targetHeight != null) { + if (animatedImage.getFrameCount() > 1) { + printWarning('targetWidth and targetHeight for multi-frame images not supported'); + } else { + animatedImage = _resizeAnimatedImage(animatedImage, targetWidth, targetHeight); + if (animatedImage == null) { + throw ImageCodecException( + 'Failed to decode re-sized image data.\n' + 'Image source: $src', + ); + } + } + } + _frameCount = animatedImage.getFrameCount().toInt(); _repetitionCount = animatedImage.getRepetitionCount().toInt(); @@ -61,6 +79,19 @@ class CkAnimatedImage extends ManagedSkiaObject return animatedImage; } + SkAnimatedImage? _resizeAnimatedImage(SkAnimatedImage animatedImage, int? targetWidth, int? targetHeight) { + final SkImage image = animatedImage.makeImageAtCurrentFrame(); + final CkImage ckImage = scaleImage(image, targetWidth, targetHeight); + final Uint8List? resizedBytes = ckImage.skImage.encodeToBytes(); + + if (resizedBytes == null) { + throw ImageCodecException('Failed to re-size image'); + } + + final SkAnimatedImage? resizedAnimatedImage = canvasKit.MakeAnimatedImageFromEncoded(resizedBytes); + return resizedAnimatedImage; + } + @override SkAnimatedImage resurrect() => createDefault(); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index a1e348e47f342..89d954900e7eb 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -44,8 +44,6 @@ void debugRestoreWebDecoderExpireDuration() { class CkBrowserImageDecoder implements ui.Codec { CkBrowserImageDecoder._({ required this.contentType, - required this.targetWidth, - required this.targetHeight, required this.data, required this.debugSource, }); @@ -53,8 +51,6 @@ class CkBrowserImageDecoder implements ui.Codec { static Future create({ required Uint8List data, required String debugSource, - int? targetWidth, - int? targetHeight, }) async { // ImageDecoder does not detect image type automatically. It requires us to // tell it what the image type is. @@ -76,8 +72,6 @@ class CkBrowserImageDecoder implements ui.Codec { final CkBrowserImageDecoder decoder = CkBrowserImageDecoder._( contentType: contentType, - targetWidth: targetWidth, - targetHeight: targetHeight, data: data, debugSource: debugSource, ); @@ -88,8 +82,6 @@ class CkBrowserImageDecoder implements ui.Codec { } final String contentType; - final int? targetWidth; - final int? targetHeight; final Uint8List data; final String debugSource; @@ -160,9 +152,6 @@ class CkBrowserImageDecoder implements ui.Codec { // Flutter always uses premultiplied alpha when decoding. premultiplyAlpha: 'premultiply', - desiredWidth: targetWidth, - desiredHeight: targetHeight, - // "default" gives the browser the liberty to convert to display-appropriate // color space, typically SRGB, which is what we want. colorSpaceConversion: 'default', diff --git a/lib/web_ui/lib/src/engine/safe_browser_api.dart b/lib/web_ui/lib/src/engine/safe_browser_api.dart index 77c2c9f8960c8..b9e900180615a 100644 --- a/lib/web_ui/lib/src/engine/safe_browser_api.dart +++ b/lib/web_ui/lib/src/engine/safe_browser_api.dart @@ -280,8 +280,8 @@ class ImageDecoderOptions { required String type, required Uint8List data, required String premultiplyAlpha, - required int? desiredWidth, - required int? desiredHeight, + int? desiredWidth, + int? desiredHeight, required String colorSpaceConversion, required bool preferAnimation, }); diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index d611a0fc8c753..08fbb721929ed 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -59,16 +59,30 @@ void testMain() { void _testForImageCodecs({required bool useBrowserImageDecoder}) { final String mode = useBrowserImageDecoder ? 'webcodecs' : 'wasm'; + final List warnings = []; + late void Function(String) oldPrintWarning; - group('($mode})', () { + group('($mode)', () { setUp(() { browserSupportsImageDecoder = useBrowserImageDecoder; + warnings.clear(); + }); + + setUpAll(() { + oldPrintWarning = printWarning; + printWarning = (String warning) { + warnings.add(warning); + }; }); tearDown(() { debugResetBrowserSupportsImageDecoder(); }); + tearDownAll(() { + printWarning = oldPrintWarning; + }); + test('CkAnimatedImage can be explicitly disposed of', () { final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kTransparentImage, 'test'); expect(image.debugDisposed, isFalse); @@ -260,9 +274,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { ); final ui.Image image = (await codec.getNextFrame()).image; - // TODO(yjbanov): https://github.com/flutter/flutter/issues/34075 - // expect(image.width, targetWidth); - // expect(image.height, targetHeight); + expect(image.width, targetWidth); + expect(image.height, targetHeight); image.dispose(); codec.dispose(); } @@ -270,6 +283,33 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { testCollector.collectNow(); }); + test('instantiateImageCodec with multi-frame image does not support targetWidth/targetHeight', + () async { + final ui.Codec codec = await ui.instantiateImageCodec( + kAnimatedGif, + targetWidth: 2, + targetHeight: 3, + ); + final ui.Image image = (await codec.getNextFrame()).image; + + expect( + warnings, + containsAllInOrder( + [ + 'targetWidth and targetHeight for multi-frame images not supported', + ], + ), + ); + + // expect the re-size did not happen, kAnimatedGif is [1x1] + expect(image.width, 1); + expect(image.height, 1); + image.dispose(); + codec.dispose(); + + testCollector.collectNow(); + }); + test('skiaInstantiateWebImageCodec throws exception on request error', () async { final TestHttpRequestMock mock = TestHttpRequestMock(); @@ -468,6 +508,64 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { expect(image2.height, 100); }); + test('decodeImageFromPixels respects target image size', () async { + Future testDecodeFromPixels(int width, int height, int targetWidth, int targetHeight) async { + final Completer completer = Completer(); + ui.decodeImageFromPixels( + Uint8List.fromList(List.filled(width * height * 4, 0)), + width, + height, + ui.PixelFormat.rgba8888, + (ui.Image image) { + completer.complete(image); + }, + targetWidth: targetWidth, + targetHeight: targetHeight, + ); + return completer.future; + } + + const List> targetSizes = >[ + [1, 1], + [1, 2], + [2, 3], + [3, 4], + [4, 4], + [10, 20], + ]; + + for (final List targetSize in targetSizes) { + final int targetWidth = targetSize[0]; + final int targetHeight = targetSize[1]; + + final ui.Image image = await testDecodeFromPixels(10, 20, targetWidth, targetHeight); + + expect(image.width, targetWidth); + expect(image.height, targetHeight); + image.dispose(); + } + }); + + test('decodeImageFromPixels upscale when allowUpscaling is false', () async { + Future testDecodeFromPixels(int width, int height) async { + final Completer completer = Completer(); + ui.decodeImageFromPixels( + Uint8List.fromList(List.filled(width * height * 4, 0)), + width, + height, + ui.PixelFormat.rgba8888, + (ui.Image image) { + completer.complete(image); + }, + targetWidth: 20, + targetHeight: 30, + allowUpscaling: false + ); + return completer.future; + } + expect(() async => testDecodeFromPixels(10, 20), throwsAssertionError); + }); + test('Decode test images', () async { final DomResponse listingResponse = await httpFetch('/test_images/'); final List testFiles = (await listingResponse.json() as List).cast();