From 30c575140a82fba8f26503069bef362f38d67e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= <737941+loic-sharma@users.noreply.github.com> Date: Tue, 6 Dec 2022 13:57:25 -0800 Subject: [PATCH] [Android] Refactor the `flutter run` Android console output test (#115023) * [Android] Refactor the flutter run Android console output test * CI bump --- dev/devicelab/bin/tasks/run_release_test.dart | 156 +----------------- dev/devicelab/lib/tasks/run_tests.dart | 110 ++++++++++++ 2 files changed, 112 insertions(+), 154 deletions(-) diff --git a/dev/devicelab/bin/tasks/run_release_test.dart b/dev/devicelab/bin/tasks/run_release_test.dart index b066f59d146b..eff5a691e52b 100644 --- a/dev/devicelab/bin/tasks/run_release_test.dart +++ b/dev/devicelab/bin/tasks/run_release_test.dart @@ -2,161 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/task_result.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:path/path.dart' as path; +import 'package:flutter_devicelab/tasks/run_tests.dart'; void main() { - task(() async { - final Device device = await devices.workingDevice; - await device.unlock(); - final Directory appDir = dir(path.join(flutterDirectory.path, 'dev/integration_tests/ui')); - await inDirectory(appDir, () async { - final Completer ready = Completer(); - final List stdout = []; - final List stderr = []; - - // Uninstall if the app is already installed on the device to get to a clean state. - print('uninstalling...'); - final Process uninstall = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['--suppress-analytics', 'install', '--uninstall-only', '-d', device.deviceId], - )..stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('uninstall:stdout: $line'); - })..stderr - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('uninstall:stderr: $line'); - stderr.add(line); - }); - if (await uninstall.exitCode != 0) { - throw 'flutter install --uninstall-only failed.'; - } - - print('run: starting...'); - final Process run = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['--suppress-analytics', 'run', '--release', '-d', device.deviceId, 'lib/main.dart'], - isBot: false, // we just want to test the output, not have any debugging info - ); - int? runExitCode; - run.stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('run:stdout: $line'); - if ( - !line.startsWith('Building flutter tool...') && - !line.startsWith('Running "flutter pub get" in ui...') && - !line.startsWith('Initializing gradle...') && - !line.contains('settings_aar.gradle') && - !line.startsWith('Resolving dependencies...') && - // Catch engine piped output from unrelated concurrent Flutter apps - !line.contains(RegExp(r'[A-Z]\/flutter \([0-9]+\):')) && - // Empty lines could be due to the progress spinner breaking up. - line.length > 1 - ) { - stdout.add(line); - } - if (line.contains('Quit (terminate the application on the device).')) { - ready.complete(); - } - }); - run.stderr - .transform(utf8.decoder) - .transform(const LineSplitter()) - // TODO(egarciad): Remove once https://github.com/flutter/flutter/issues/95131 is fixed. - .skipWhile((String line) => line.contains('Mapping new ns')) - .listen((String line) { - print('run:stderr: $line'); - stderr.add(line); - }); - unawaited(run.exitCode.then((int exitCode) { runExitCode = exitCode; })); - await Future.any(>[ ready.future, run.exitCode ]); - if (runExitCode != null) { - throw 'Failed to run test app; runner unexpected exited, with exit code $runExitCode.'; - } - run.stdin.write('q'); - - await run.exitCode; - - if (stderr.isNotEmpty) { - throw 'flutter run --release had output on standard error.'; - } - - _findNextMatcherInList( - stdout, - (String line) => line.startsWith('Launching lib/main.dart on ') && line.endsWith(' in release mode...'), - 'Launching lib/main.dart on', - ); - - _findNextMatcherInList( - stdout, - (String line) => line.startsWith("Running Gradle task 'assembleRelease'..."), - "Running Gradle task 'assembleRelease'...", - ); - - _findNextMatcherInList( - stdout, - (String line) => line.contains('Built build/app/outputs/flutter-apk/app-release.apk (') && line.contains('MB).'), - 'Built build/app/outputs/flutter-apk/app-release.apk', - ); - - _findNextMatcherInList( - stdout, - (String line) => line.startsWith('Installing build/app/outputs/flutter-apk/app-release.apk...'), - 'Installing build/app/outputs/flutter-apk/app-release.apk...', - ); - - _findNextMatcherInList( - stdout, - (String line) => line.contains('Quit (terminate the application on the device).'), - 'q Quit (terminate the application on the device)', - ); - - _findNextMatcherInList( - stdout, - (String line) => line == 'Application finished.', - 'Application finished.', - ); - }); - return TaskResult.success(null); - }); -} - -void _findNextMatcherInList( - List list, - bool Function(String testLine) matcher, - String errorMessageExpectedLine -) { - final List copyOfListForErrorMessage = List.from(list); - - while (list.isNotEmpty) { - final String nextLine = list.first; - list.removeAt(0); - - if (matcher(nextLine)) { - return; - } - } - - throw ''' -Did not find expected line - -$errorMessageExpectedLine - -in flutter run --release stdout - -$copyOfListForErrorMessage - '''; + task(createAndroidRunReleaseTest()); } diff --git a/dev/devicelab/lib/tasks/run_tests.dart b/dev/devicelab/lib/tasks/run_tests.dart index 1fb94f59191b..96db35f72b75 100644 --- a/dev/devicelab/lib/tasks/run_tests.dart +++ b/dev/devicelab/lib/tasks/run_tests.dart @@ -11,6 +11,10 @@ import '../framework/framework.dart'; import '../framework/task_result.dart'; import '../framework/utils.dart'; +TaskFunction createAndroidRunReleaseTest() { + return AndroidRunOutputTest(release: true); +} + TaskFunction createMacOSRunReleaseTest() { return DesktopRunOutputTest( // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201 @@ -18,17 +22,110 @@ TaskFunction createMacOSRunReleaseTest() { '${flutterDirectory.path}/examples/hello_world', 'lib/main.dart', release: true, + allowStderr: true, ); } +class AndroidRunOutputTest extends RunOutputTask { + AndroidRunOutputTest({required super.release}) : super( + '${flutterDirectory.path}/dev/integration_tests/ui', + 'lib/main.dart', + ); + + @override + Future prepare(String deviceId) async { + // Uninstall if the app is already installed on the device to get to a clean state. + final List stderr = []; + print('uninstalling...'); + final Process uninstall = await startFlutter( + 'install', + options: ['--suppress-analytics', '--uninstall-only', '-d', deviceId], + isBot: false, + ); + uninstall.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('uninstall:stdout: $line'); + }); + uninstall.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + print('uninstall:stderr: $line'); + stderr.add(line); + }); + if (await uninstall.exitCode != 0) { + throw 'flutter install --uninstall-only failed.'; + } + if (stderr.isNotEmpty) { + throw 'flutter install --uninstall-only had output on standard error.'; + } + } + + @override + bool isExpectedStderr(String line) { + // TODO(egarciad): Remove once https://github.com/flutter/flutter/issues/95131 is fixed. + return line.contains('Mapping new ns'); + } + + @override + TaskResult verify(List stdout, List stderr) { + _findNextMatcherInList( + stdout, + (String line) => line.startsWith('Launching lib/main.dart on ') && line.endsWith(' in release mode...'), + 'Launching lib/main.dart on', + ); + + _findNextMatcherInList( + stdout, + (String line) => line.startsWith("Running Gradle task 'assembleRelease'..."), + "Running Gradle task 'assembleRelease'...", + ); + + _findNextMatcherInList( + stdout, + (String line) => line.contains('Built build/app/outputs/flutter-apk/app-release.apk (') && line.contains('MB).'), + 'Built build/app/outputs/flutter-apk/app-release.apk', + ); + + _findNextMatcherInList( + stdout, + (String line) => line.startsWith('Installing build/app/outputs/flutter-apk/app-release.apk...'), + 'Installing build/app/outputs/flutter-apk/app-release.apk...', + ); + + _findNextMatcherInList( + stdout, + (String line) => line.contains('Quit (terminate the application on the device).'), + 'q Quit (terminate the application on the device)', + ); + + _findNextMatcherInList( + stdout, + (String line) => line == 'Application finished.', + 'Application finished.', + ); + + return TaskResult.success(null); + } +} + class DesktopRunOutputTest extends RunOutputTask { DesktopRunOutputTest( super.testDirectory, super.testTarget, { required super.release, + this.allowStderr = false, } ); + /// Whether `flutter run` is expected to produce output on stderr. + final bool allowStderr; + + @override + bool isExpectedStderr(String line) => allowStderr; + @override TaskResult verify(List stdout, List stderr) { _findNextMatcherInList( @@ -80,6 +177,8 @@ abstract class RunOutputTask { final List stdout = []; final List stderr = []; + await prepare(deviceId); + final List options = [ testTarget, '-d', @@ -107,6 +206,7 @@ abstract class RunOutputTask { run.stderr .transform(utf8.decoder) .transform(const LineSplitter()) + .skipWhile(isExpectedStderr) .listen((String line) { print('run:stderr: $line'); stderr.add(line); @@ -120,10 +220,20 @@ abstract class RunOutputTask { await run.exitCode; + if (stderr.isNotEmpty) { + throw 'flutter run ${release ? '--release' : ''} had unexpected output on standard error.'; + } + return verify(stdout, stderr); }); } + /// Prepare the device for running the test app. + Future prepare(String deviceId) => Future.value(); + + /// Returns true if this stderr output line is expected. + bool isExpectedStderr(String line) => false; + /// Verify the output of `flutter run`. TaskResult verify(List stdout, List stderr) => throw UnimplementedError('verify is not implemented');