diff --git a/script/tool/lib/src/common/process_runner.dart b/script/tool/lib/src/common/process_runner.dart index 429761ead3b..7556b559c52 100644 --- a/script/tool/lib/src/common/process_runner.dart +++ b/script/tool/lib/src/common/process_runner.dart @@ -36,8 +36,10 @@ class ProcessRunner { 'Running command: "$executable ${args.join(' ')}" in ${workingDir?.path ?? io.Directory.current.path}'); final io.Process process = await io.Process.start(executable, args, workingDirectory: workingDir?.path); - await io.stdout.addStream(process.stdout); - await io.stderr.addStream(process.stderr); + await Future.wait(>[ + io.stdout.addStream(process.stdout), + io.stderr.addStream(process.stderr), + ]); if (exitOnError && await process.exitCode != 0) { final String error = _getErrorString(executable, args, workingDir: workingDir); diff --git a/script/tool/lib/src/drive_examples_command.dart b/script/tool/lib/src/drive_examples_command.dart index 279fdce2fd4..9bbfd354b37 100644 --- a/script/tool/lib/src/drive_examples_command.dart +++ b/script/tool/lib/src/drive_examples_command.dart @@ -17,7 +17,7 @@ import 'common/repository_package.dart'; const int _exitNoPlatformFlags = 2; const int _exitNoAvailableDevice = 3; -/// A command to run the example applications for packages via Flutter driver. +/// A command to run the integration tests for a package's example applications. class DriveExamplesCommand extends PackageLoopingCommand { /// Creates an instance of the drive command. DriveExamplesCommand( @@ -50,11 +50,9 @@ class DriveExamplesCommand extends PackageLoopingCommand { final String name = 'drive-examples'; @override - final String description = 'Runs driver tests for package example apps.\n\n' - 'For each *_test.dart in test_driver/ it drives an application with ' - 'either the corresponding test in test_driver (for example, ' - 'test_driver/app_test.dart would match test_driver/app.dart), or the ' - '*_test.dart files in integration_test/.\n\n' + final String description = 'Runs Dart integration tests for example apps.\n\n' + "This runs all tests in each example's integration_test directory, " + 'via "flutter test" on most platforms, and "flutter drive" on web.\n\n' 'This command requires "flutter" to be in your path.'; Map> _targetDeviceFlags = const >{}; @@ -164,53 +162,53 @@ class DriveExamplesCommand extends PackageLoopingCommand { 'Skipping $exampleName; does not support any requested platforms.'); continue; } + ++supportedExamplesFound; - final List drivers = await _getDrivers(example); - if (drivers.isEmpty) { - print('No driver tests found for $exampleName'); + final List testTargets = await _getIntegrationTests(example); + if (testTargets.isEmpty) { + print('No integration_test/*.dart files found for $exampleName.'); continue; } - for (final File driver in drivers) { - final List testTargets = []; - - // Try to find a matching app to drive without the _test.dart - // TODO(stuartmorgan): Migrate all remaining uses of this legacy - // approach (currently only video_player) and remove support for it: - // https://github.com/flutter/flutter/issues/85224. - final File? legacyTestFile = _getLegacyTestFileForTestDriver(driver); - if (legacyTestFile != null) { - testTargets.add(legacyTestFile); - } else { - for (final File testFile in await _getIntegrationTests(example)) { - // Check files for known problematic patterns. - final bool passesValidation = _validateIntegrationTest(testFile); - if (!passesValidation) { - // Report the issue, but continue with the test as the validation - // errors don't prevent running. - errors.add('${testFile.basename} failed validation'); - } - testTargets.add(testFile); - } - } - - if (testTargets.isEmpty) { - final String driverRelativePath = - getRelativePosixPath(driver, from: package.directory); - printError( - 'Found $driverRelativePath, but no integration_test/*_test.dart files.'); - errors.add('No test files for $driverRelativePath'); + // Check files for known problematic patterns. + testTargets + .where((File file) => !_validateIntegrationTest(file)) + .forEach((File file) { + // Report the issue, but continue with the test as the validation + // errors don't prevent running. + errors.add('${file.basename} failed validation'); + }); + + // `flutter test` doesn't yet support web integration tests, so fall back + // to `flutter drive`. + final bool useFlutterDrive = getBoolArg(platformWeb); + + final List drivers; + if (useFlutterDrive) { + drivers = await _getDrivers(example); + if (drivers.isEmpty) { + print('No driver found for $exampleName'); continue; } + } else { + drivers = []; + } - testsRan = true; - final List failingTargets = await _driveTests( - example, driver, testTargets, - deviceFlags: deviceFlags); - for (final File failingTarget in failingTargets) { - errors.add( - getRelativePosixPath(failingTarget, from: package.directory)); + testsRan = true; + if (useFlutterDrive) { + for (final File driver in drivers) { + final List failingTargets = await _driveTests( + example, driver, testTargets, + deviceFlags: deviceFlags); + for (final File failingTarget in failingTargets) { + errors.add( + getRelativePosixPath(failingTarget, from: package.directory)); + } + } + } else { + if (!await _runTests(example, deviceFlags: deviceFlags)) { + errors.add('Integration tests failed.'); } } } @@ -224,7 +222,7 @@ class DriveExamplesCommand extends PackageLoopingCommand { } else { return PackageResult.skip(supportedExamplesFound == 0 ? 'No example supports requested platform(s).' - : 'No example is configured for driver tests.'); + : 'No example is configured for integration tests.'); } } return errors.isEmpty @@ -295,16 +293,6 @@ class DriveExamplesCommand extends PackageLoopingCommand { return drivers; } - File? _getLegacyTestFileForTestDriver(File testDriver) { - final String testName = testDriver.basename.replaceAll( - RegExp(r'_test.dart$'), - '.dart', - ); - final File testFile = testDriver.parent.childFile(testName); - - return testFile.existsSync() ? testFile : null; - } - Future> _getIntegrationTests(RepositoryPackage example) async { final List tests = []; final Directory integrationTestDir = @@ -378,4 +366,31 @@ class DriveExamplesCommand extends PackageLoopingCommand { } return failures; } + + /// Uses `flutter test integration_test` to run [example], returning the + /// success of the test run. + /// + /// [deviceFlags] should contain the flags to run the test on a specific + /// target device (plus any supporting device-specific flags). E.g.: + /// - `['-d', 'macos']` for driving for macOS. + /// - `['-d', 'web-server', '--web-port=', '--browser-name=]` + /// for web + Future _runTests( + RepositoryPackage example, { + required List deviceFlags, + }) async { + final String enableExperiment = getStringArg(kEnableExperiment); + + final int exitCode = await processRunner.runAndStream( + flutterCommand, + [ + 'test', + ...deviceFlags, + if (enableExperiment.isNotEmpty) + '--enable-experiment=$enableExperiment', + 'integration_test', + ], + workingDir: example.directory); + return exitCode == 0; + } } diff --git a/script/tool/test/drive_examples_command_test.dart b/script/tool/test/drive_examples_command_test.dart index 81056db7c8e..203d525a552 100644 --- a/script/tool/test/drive_examples_command_test.dart +++ b/script/tool/test/drive_examples_command_test.dart @@ -190,91 +190,6 @@ void main() { ); }); - test('driving under folder "test_driver"', () async { - final RepositoryPackage plugin = createFakePlugin( - 'plugin', - packagesDir, - extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', - 'example/android/android.java', - 'example/ios/ios.m', - ], - platformSupport: { - platformAndroid: const PlatformDetails(PlatformSupport.inline), - platformIOS: const PlatformDetails(PlatformSupport.inline), - }, - ); - - final Directory pluginExampleDirectory = getExampleDir(plugin); - - setMockFlutterDevicesOutput(); - final List output = - await runCapturingPrint(runner, ['drive-examples', '--ios']); - - expect( - output, - containsAllInOrder([ - contains('Running for plugin'), - contains('No issues found!'), - ]), - ); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall(getFlutterCommand(mockPlatform), - const ['devices', '--machine'], null), - ProcessCall( - getFlutterCommand(mockPlatform), - const [ - 'drive', - '-d', - _fakeIOSDevice, - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' - ], - pluginExampleDirectory.path), - ])); - }); - - test('driving under folder "test_driver" when test files are missing"', - () async { - setMockFlutterDevicesOutput(); - createFakePlugin( - 'plugin', - packagesDir, - extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/android/android.java', - 'example/ios/ios.m', - ], - platformSupport: { - platformAndroid: const PlatformDetails(PlatformSupport.inline), - platformIOS: const PlatformDetails(PlatformSupport.inline), - }, - ); - - Error? commandError; - final List output = await runCapturingPrint( - runner, ['drive-examples', '--android'], - errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - expect( - output, - containsAllInOrder([ - contains('Running for plugin'), - contains('No driver tests were run (1 example(s) found).'), - contains('No test files for example/test_driver/plugin_test.dart'), - ]), - ); - }); - test('a plugin without any integration test files is reported as an error', () async { setMockFlutterDevicesOutput(); @@ -351,14 +266,11 @@ void main() { ); }); - test( - 'driving under folder "test_driver" when targets are under "integration_test"', - () async { + test('tests an iOS plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/integration_test.dart', 'example/integration_test/bar_test.dart', 'example/integration_test/foo_test.dart', 'example/integration_test/ignore_me.dart', @@ -393,25 +305,10 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', _fakeIOSDevice, - '--driver', - 'test_driver/integration_test.dart', - '--target', - 'integration_test/bar_test.dart', - ], - pluginExampleDirectory.path), - ProcessCall( - getFlutterCommand(mockPlatform), - const [ - 'drive', - '-d', - _fakeIOSDevice, - '--driver', - 'test_driver/integration_test.dart', - '--target', - 'integration_test/foo_test.dart', + 'integration_test', ], pluginExampleDirectory.path), ])); @@ -419,8 +316,7 @@ void main() { test('driving when plugin does not support Linux is a no-op', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ]); final List output = await runCapturingPrint(runner, [ @@ -442,13 +338,12 @@ void main() { expect(processRunner.recordedCalls, []); }); - test('driving on a Linux plugin', () async { + test('tests a Linux plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/linux/linux.cc', ], platformSupport: { @@ -477,13 +372,10 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', 'linux', - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); @@ -491,8 +383,7 @@ void main() { test('driving when plugin does not suppport macOS is a no-op', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ]); final List output = await runCapturingPrint(runner, [ @@ -514,13 +405,12 @@ void main() { expect(processRunner.recordedCalls, []); }); - test('driving on a macOS plugin', () async { + test('tests a macOS plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/macos/macos.swift', ], platformSupport: { @@ -549,13 +439,10 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', 'macos', - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); @@ -563,8 +450,7 @@ void main() { test('driving when plugin does not suppport web is a no-op', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ]); final List output = await runCapturingPrint(runner, [ @@ -585,13 +471,13 @@ void main() { expect(processRunner.recordedCalls, []); }); - test('driving a web plugin', () async { + test('drives a web plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', + 'example/test_driver/integration_test.dart', 'example/web/index.html', ], platformSupport: { @@ -626,21 +512,21 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--driver', - 'test_driver/plugin_test.dart', + 'test_driver/integration_test.dart', '--target', - 'test_driver/plugin.dart' + 'integration_test/plugin_test.dart', ], pluginExampleDirectory.path), ])); }); - test('driving a web plugin with CHROME_EXECUTABLE', () async { + test('drives a web plugin with CHROME_EXECUTABLE', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', + 'example/test_driver/integration_test.dart', 'example/web/index.html', ], platformSupport: { @@ -678,9 +564,9 @@ void main() { '--browser-name=chrome', '--chrome-binary=/path/to/chrome', '--driver', - 'test_driver/plugin_test.dart', + 'test_driver/integration_test.dart', '--target', - 'test_driver/plugin.dart' + 'integration_test/plugin_test.dart', ], pluginExampleDirectory.path), ])); @@ -688,8 +574,7 @@ void main() { test('driving when plugin does not suppport Windows is a no-op', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ]); final List output = await runCapturingPrint(runner, [ @@ -711,13 +596,12 @@ void main() { expect(processRunner.recordedCalls, []); }); - test('driving on a Windows plugin', () async { + test('tests a Windows plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/windows/windows.cpp', ], platformSupport: { @@ -746,25 +630,21 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', 'windows', - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); }); - test('driving on an Android plugin', () async { + test('tests an Android plugin', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/android/android.java', ], platformSupport: { @@ -796,25 +676,21 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', _fakeAndroidDevice, - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); }); - test('driving on an Android plugin with alias', () async { + test('tests an Android plugin with "apk" alias', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/android/android.java', ], platformSupport: { @@ -846,13 +722,10 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', _fakeAndroidDevice, - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); @@ -863,8 +736,7 @@ void main() { 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ], platformSupport: { platformMacOS: const PlatformDetails(PlatformSupport.inline), @@ -896,8 +768,7 @@ void main() { 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', ], platformSupport: { platformMacOS: const PlatformDetails(PlatformSupport.inline), @@ -951,8 +822,7 @@ void main() { 'plugin', packagesDir, extraFiles: [ - 'example/test_driver/plugin_test.dart', - 'example/test_driver/plugin.dart', + 'example/integration_test/plugin_test.dart', 'example/android/android.java', 'example/ios/ios.m', ], @@ -979,14 +849,11 @@ void main() { ProcessCall( getFlutterCommand(mockPlatform), const [ - 'drive', + 'test', '-d', _fakeIOSDevice, '--enable-experiment=exp1', - '--driver', - 'test_driver/plugin_test.dart', - '--target', - 'test_driver/plugin.dart' + 'integration_test', ], pluginExampleDirectory.path), ])); @@ -1021,7 +888,7 @@ void main() { ); }); - test('fails when no driver is present', () async { + test('web fails when no driver is present', () async { createFakePlugin( 'plugin', packagesDir, @@ -1046,7 +913,7 @@ void main() { output, containsAllInOrder([ contains('Running for plugin'), - contains('No driver tests found for plugin/example'), + contains('No driver found for plugin/example'), contains('No driver tests were run (1 example(s) found).'), contains('The following packages had errors:'), contains(' plugin:\n' @@ -1055,7 +922,7 @@ void main() { ); }); - test('fails when no integration tests are present', () async { + test('web fails when no integration tests are present', () async { createFakePlugin( 'plugin', packagesDir, @@ -1079,18 +946,15 @@ void main() { output, containsAllInOrder([ contains('Running for plugin'), - contains('Found example/test_driver/integration_test.dart, but no ' - 'integration_test/*_test.dart files.'), contains('No driver tests were run (1 example(s) found).'), contains('The following packages had errors:'), contains(' plugin:\n' - ' No test files for example/test_driver/integration_test.dart\n' ' No tests ran (use --exclude if this is intentional)'), ]), ); }); - test('reports test failures', () async { + test('"flutter drive" reports test failures', () async { final RepositoryPackage plugin = createFakePlugin( 'plugin', packagesDir, @@ -1098,10 +962,10 @@ void main() { 'example/test_driver/integration_test.dart', 'example/integration_test/bar_test.dart', 'example/integration_test/foo_test.dart', - 'example/macos/macos.swift', + 'example/web/index.html', ], platformSupport: { - platformMacOS: const PlatformDetails(PlatformSupport.inline), + platformWeb: const PlatformDetails(PlatformSupport.inline), }, ); @@ -1109,17 +973,14 @@ void main() { processRunner .mockProcessesForExecutable[getFlutterCommand(mockPlatform)] = [ - // No mock for 'devices', since it's running for macOS. - FakeProcessInfo( - MockProcess(exitCode: 1), ['drive']), // 'drive' #1 - FakeProcessInfo( - MockProcess(exitCode: 1), ['drive']), // 'drive' #2 + // Fail both bar_test.dart and foo_test.dart. + FakeProcessInfo(MockProcess(exitCode: 1), ['drive']), + FakeProcessInfo(MockProcess(exitCode: 1), ['drive']), ]; Error? commandError; - final List output = - await runCapturingPrint(runner, ['drive-examples', '--macos'], - errorHandler: (Error e) { + final List output = await runCapturingPrint( + runner, ['drive-examples', '--web'], errorHandler: (Error e) { commandError = e; }); @@ -1144,7 +1005,9 @@ void main() { const [ 'drive', '-d', - 'macos', + 'web-server', + '--web-port=7357', + '--browser-name=chrome', '--driver', 'test_driver/integration_test.dart', '--target', @@ -1156,7 +1019,9 @@ void main() { const [ 'drive', '-d', - 'macos', + 'web-server', + '--web-port=7357', + '--browser-name=chrome', '--driver', 'test_driver/integration_test.dart', '--target', @@ -1166,6 +1031,61 @@ void main() { ])); }); + test('"flutter test" reports test failures', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin', + packagesDir, + extraFiles: [ + 'example/integration_test/bar_test.dart', + 'example/integration_test/foo_test.dart', + 'example/macos/macos.swift', + ], + platformSupport: { + platformMacOS: const PlatformDetails(PlatformSupport.inline), + }, + ); + + // Simulate failure from `flutter test`. + processRunner + .mockProcessesForExecutable[getFlutterCommand(mockPlatform)] = + [ + FakeProcessInfo(MockProcess(exitCode: 1), ['test']), + ]; + + Error? commandError; + final List output = + await runCapturingPrint(runner, ['drive-examples', '--macos'], + errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('The following packages had errors:'), + contains(' plugin:\n' + ' Integration tests failed.'), + ]), + ); + + final Directory pluginExampleDirectory = getExampleDir(plugin); + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + getFlutterCommand(mockPlatform), + const [ + 'test', + '-d', + 'macos', + 'integration_test', + ], + pluginExampleDirectory.path), + ])); + }); + group('packages', () { test('can be driven', () async { final RepositoryPackage package = @@ -1301,7 +1221,8 @@ void main() { output, containsAllInOrder([ contains('Running for a_package'), - contains('SKIPPING: No example is configured for driver tests.'), + contains( + 'SKIPPING: No example is configured for integration tests.'), ]), );