From 41625b66209f61b7c23dc596da910816d689ea61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= <737941+loic-sharma@users.noreply.github.com> Date: Mon, 12 Dec 2022 14:39:49 -0800 Subject: [PATCH] Test flutter run does not have unexpected engine logs (#116798) --- .ci.yaml | 80 ++++++++++++++++++- TESTOWNERS | 3 + dev/devicelab/README.md | 2 +- .../bin/tasks/run_debug_test_android.dart | 10 +++ .../bin/tasks/run_debug_test_macos.dart | 12 +++ .../bin/tasks/run_debug_test_windows.dart | 12 +++ dev/devicelab/lib/tasks/run_tests.dart | 34 ++++++++ 7 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 dev/devicelab/bin/tasks/run_debug_test_android.dart create mode 100644 dev/devicelab/bin/tasks/run_debug_test_macos.dart create mode 100644 dev/devicelab/bin/tasks/run_debug_test_windows.dart diff --git a/.ci.yaml b/.ci.yaml index 7d92df7395d1..7914a7da8e81 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -3206,6 +3206,30 @@ targets: ["devicelab", "android", "mac"] task_name: microbenchmarks + - name: Mac_android run_debug_test_android + recipe: devicelab/devicelab_drone + bringup: true + presubmit: false + runIf: + - dev/** + timeout: 60 + properties: + tags: > + ["devicelab", "android", "mac"] + task_name: run_debug_test_android + + - name: Mac_arm64_android run_debug_test_android + recipe: devicelab/devicelab_drone + bringup: true + presubmit: false + runIf: + - dev/** + timeout: 60 + properties: + tags: > + ["devicelab", "android", "mac", "arm64"] + task_name: run_debug_test_android + - name: Mac_android run_release_test recipe: devicelab/devicelab_drone presubmit: false @@ -3879,8 +3903,42 @@ targets: - bin/** - .ci.yaml + - name: Mac run_debug_test_macos + recipe: devicelab/devicelab_drone + bringup: true + timeout: 60 + properties: + dependencies: >- + [ + {"dependency": "xcode", "version": "14a5294e"}, + {"dependency": "gems", "version": "v3.3.14"} + ] + tags: > + ["devicelab", "hostonly", "mac"] + task_name: run_debug_test_macos + runIf: + - dev/** + - packages/flutter_tools/** + - bin/** + - .ci.yaml + + - name: Mac_arm64_ios run_debug_test_macos + recipe: devicelab/devicelab_drone + bringup: true + timeout: 60 + properties: + tags: > + ["devicelab", "ios", "mac", "arm64"] + task_name: run_debug_test_macos + runIf: + - dev/** + - packages/flutter_tools/** + - bin/** + - .ci.yaml + - name: Mac run_release_test_macos recipe: devicelab/devicelab_drone + presubmit: false timeout: 60 properties: dependencies: >- @@ -4193,9 +4251,29 @@ targets: - bin/** - .ci.yaml - - name: Windows run_release_test_windows + - name: Windows run_debug_test_windows + recipe: devicelab/devicelab_drone bringup: true + presubmit: false + timeout: 60 + properties: + dependencies: >- + [ + {"dependency": "vs_build", "version": "version:vs2019"} + ] + tags: > + ["devicelab", "hostonly", "windows"] + task_name: run_debug_test_windows + runIf: + - dev/** + - packages/flutter_tools/** + - bin/** + - .ci.yaml + + - name: Windows run_release_test_windows recipe: devicelab/devicelab_drone + bringup: true + presubmit: false timeout: 60 properties: dependencies: >- diff --git a/TESTOWNERS b/TESTOWNERS index 0043b10ae757..60fac112cf25 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -249,6 +249,9 @@ /dev/devicelab/bin/tasks/plugin_lint_mac.dart @stuartmorgan @flutter/plugin /dev/devicelab/bin/tasks/plugin_test_ios.dart @jmagman @flutter/ios /dev/devicelab/bin/tasks/plugin_test.dart @stuartmorgan @flutter/plugin +/dev/devicelab/bin/tasks/run_debug_test_android.dart @zanderso @flutter/tool +/dev/devicelab/bin/tasks/run_debug_test_macos.dart @cbracken @flutter/tool +/dev/devicelab/bin/tasks/run_debug_test_windows.dart @loic-sharma @flutter/tool /dev/devicelab/bin/tasks/run_release_test_macos.dart @cbracken @flutter/tool /dev/devicelab/bin/tasks/run_release_test_windows.dart @loic-sharma @flutter/tool /dev/devicelab/bin/tasks/technical_debt__cost.dart @HansMuller @flutter/framework diff --git a/dev/devicelab/README.md b/dev/devicelab/README.md index d6e75d7d3cf4..760d640d0fcd 100644 --- a/dev/devicelab/README.md +++ b/dev/devicelab/README.md @@ -44,7 +44,7 @@ You must set the `ANDROID_SDK_ROOT` environment variable to run tests on Android. If you have a local build of the Flutter engine, then you have a copy of the Android SDK at `.../engine/src/third_party/android_tools/sdk`. -You can find where your Android SDK is using `flutter doctor`. +You can find where your Android SDK is using `flutter doctor -v`. ### Warnings diff --git a/dev/devicelab/bin/tasks/run_debug_test_android.dart b/dev/devicelab/bin/tasks/run_debug_test_android.dart new file mode 100644 index 000000000000..93ac75612e75 --- /dev/null +++ b/dev/devicelab/bin/tasks/run_debug_test_android.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/run_tests.dart'; + +void main() { + task(createAndroidRunDebugTest()); +} diff --git a/dev/devicelab/bin/tasks/run_debug_test_macos.dart b/dev/devicelab/bin/tasks/run_debug_test_macos.dart new file mode 100644 index 000000000000..7fc3eaf1e309 --- /dev/null +++ b/dev/devicelab/bin/tasks/run_debug_test_macos.dart @@ -0,0 +1,12 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/run_tests.dart'; + +void main() { + deviceOperatingSystem = DeviceOperatingSystem.macos; + task(createMacOSRunDebugTest()); +} diff --git a/dev/devicelab/bin/tasks/run_debug_test_windows.dart b/dev/devicelab/bin/tasks/run_debug_test_windows.dart new file mode 100644 index 000000000000..bff954ccead4 --- /dev/null +++ b/dev/devicelab/bin/tasks/run_debug_test_windows.dart @@ -0,0 +1,12 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/run_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.windows; + await task(createWindowsRunDebugTest()); +} diff --git a/dev/devicelab/lib/tasks/run_tests.dart b/dev/devicelab/lib/tasks/run_tests.dart index bec82e55a2ce..cbf1c81d26ef 100644 --- a/dev/devicelab/lib/tasks/run_tests.dart +++ b/dev/devicelab/lib/tasks/run_tests.dart @@ -11,10 +11,25 @@ import '../framework/framework.dart'; import '../framework/task_result.dart'; import '../framework/utils.dart'; +TaskFunction createAndroidRunDebugTest() { + return AndroidRunOutputTest(release: false); +} + TaskFunction createAndroidRunReleaseTest() { return AndroidRunOutputTest(release: true); } +TaskFunction createMacOSRunDebugTest() { + return DesktopRunOutputTest( + // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201 + // Switch to dev/integration_tests/ui once we have CocoaPods working on M1 Macs. + '${flutterDirectory.path}/examples/hello_world', + 'lib/main.dart', + release: false, + allowStderr: true, + ); +} + TaskFunction createMacOSRunReleaseTest() { return DesktopRunOutputTest( // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201 @@ -26,6 +41,14 @@ TaskFunction createMacOSRunReleaseTest() { ); } +TaskFunction createWindowsRunDebugTest() { + return DesktopRunOutputTest( + '${flutterDirectory.path}/dev/integration_tests/ui', + 'lib/empty.dart', + release: false, + ); +} + TaskFunction createWindowsRunReleaseTest() { return DesktopRunOutputTest( '${flutterDirectory.path}/dev/integration_tests/ui', @@ -168,6 +191,10 @@ abstract class RunOutputTask { } ); + static final RegExp _engineLogRegex = RegExp( + r'\[(VERBOSE|INFO|WARNING|ERROR|FATAL):.+\(\d+\)\]', + ); + /// The directory where the app under test is defined. final String testDirectory; /// The main entry-point file of the application, as run on the device. @@ -232,6 +259,13 @@ abstract class RunOutputTask { throw 'flutter run ${release ? '--release' : ''} had unexpected output on standard error.'; } + final List engineLogs = List.from( + stdout.where(_engineLogRegex.hasMatch), + ); + if (engineLogs.isNotEmpty) { + throw 'flutter run had unexpected Flutter engine logs $engineLogs'; + } + return verify(stdout, stderr); }); }