From bc10ee1c361e26820ebab6ad3034638292a6ca81 Mon Sep 17 00:00:00 2001 From: Godofredo Contreras Date: Wed, 24 Feb 2021 14:45:38 -0800 Subject: [PATCH] Move a second batch of mac/ios tests to prod. Bug: https://github.com/flutter/flutter/issues/73392 --- dev/devicelab/manifest.yaml | 129 -------------------------- dev/devicelab/test/manifest_test.dart | 8 +- dev/prod_builders.json | 42 ++++----- 3 files changed, 25 insertions(+), 154 deletions(-) diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 7d41b9a0058f..df81fc95baeb 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -73,18 +73,6 @@ tasks: required_agent_capabilities: ["mac/ios32"] flaky: true - ios_defines_test: - description: > - Builds a Framework with a --dart-define and verifies it can be used as a constant - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - tiles_scroll_perf_ios__timeline_summary: - description: > - Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - flutter_gallery_ios32__start_up: description: > Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S). @@ -98,36 +86,6 @@ tasks: required_agent_capabilities: ["mac/ios32"] flaky: true - platform_interaction_test_ios: - description: > - Checks platform interaction on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_channel_sample_test_ios: - description: > - Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_channel_sample_test_swift: - description: > - Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_view_ios__start_up: - description: > - Verifies that Platform View can be used from an iOS project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - post_backdrop_filter_perf_ios__timeline_summary: - description: > - Measures the runtime performance of animations after a backdrop filter is removed on iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - flutter_gallery_ios__start_up: description: > Measures the startup time of the Flutter Gallery app on iPhone 6. @@ -141,87 +99,6 @@ tasks: stage: devicelab_ios required_agent_capabilities: ["mac/ios"] - hello_world_ios__compile: - description: > - Measures the IPA size of Hello World. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - microbenchmarks_ios: - description: > - Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6. - stage: devicelab_ios - #TODO(godofredoc): Remove ios/12 capability once - # https://github.com/flutter/flutter/issues/49635 is fixed. - required_agent_capabilities: ["mac/ios", "ios/12"] - - flutter_view_ios__start_up: - description: > - Verifies that Flutter View can be used from an iOS project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_driver: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_keyboard_resize: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_screenshot: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_textfield: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - ios_platform_view_tests: - description: > - Runs end-to-end tests with platform views in the scene. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - macos_chrome_dev_mode: - description: > - Run flutter web on the devicelab and hot restart. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - simple_animation_perf_ios: - description: > - Measure CPU/GPU usage percentages of a simple animation. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - smoke_catalina_hot_mode_dev_cycle_ios__benchmark: - description: > - A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking. - stage: devicelab_ios - required_agent_capabilities: ["mac-catalina/ios"] - - # macOS target platform tests - hot_mode_dev_cycle_macos_target__benchmark: - description: > - Checks the functionality and performance of hot reload on a macOS target platform - stage: devicelab - required_agent_capabilities: ["mac/ios"] - - ios_app_with_extensions_test: - description: > - Checks that an iOS app with extensions can be built for physical and simulated devices. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - # Tests running on Windows host flavors_test_win: @@ -264,9 +141,3 @@ tasks: Measures the performance of screen transitions in the new Flutter Gallery on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] - - large_image_changer_perf_ios: - description: > - Measures memory, cpu, and gpu usage when rotating through a series of large images. - stage: devicelab - required_agent_capabilities: ["mac/ios"] diff --git a/dev/devicelab/test/manifest_test.dart b/dev/devicelab/test/manifest_test.dart index c0cc1724ed8f..005afeb1071c 100644 --- a/dev/devicelab/test/manifest_test.dart +++ b/dev/devicelab/test/manifest_test.dart @@ -14,10 +14,10 @@ void main() { final Manifest manifest = loadTaskManifest(); expect(manifest.tasks, isNotEmpty); - final ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'simple_animation_perf_ios'); - expect(task.description, 'Measure CPU/GPU usage percentages of a simple animation.\n'); - expect(task.stage, 'devicelab_ios'); - expect(task.requiredAgentCapabilities, ['mac/ios']); + final ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'complex_layout_win__compile'); + expect(task.description, 'Collects various performance metrics of compiling the Complex Layout for Android from Windows.\n'); + expect(task.stage, 'devicelab_win'); + expect(task.requiredAgentCapabilities, ['windows/android']); for (final ManifestTask task in manifest.tasks) { final File taskFile = File('bin/tasks/${task.name}.dart'); diff --git a/dev/prod_builders.json b/dev/prod_builders.json index 4507ee521a5b..7cc4d8a0ee7c 100644 --- a/dev/prod_builders.json +++ b/dev/prod_builders.json @@ -874,49 +874,49 @@ "name": "Mac_ios flutter_view_ios__start_up", "repo": "flutter", "task_name": "mac_ios_flutter_view_ios__start_up", - "flaky": true + "flaky": false }, { "name": "Mac_ios hello_world_ios__compile", "repo": "flutter", "task_name": "mac_ios_hello_world_ios__compile", - "flaky": true + "flaky": false }, { "name": "Mac_ios hot_mode_dev_cycle_macos_target__benchmark", "repo": "flutter", "task_name": "mac_ios_hot_mode_dev_cycle_macos_target__benchmark", - "flaky": true + "flaky": false }, { "name": "Mac_ios integration_ui_ios_driver", "repo": "flutter", "task_name": "mac_ios_integration_ui_ios_driver", - "flaky": true + "flaky": false }, { "name": "Mac_ios integration_ui_ios_keyboard_resize", "repo": "flutter", "task_name": "mac_ios_integration_ui_ios_keyboard_resize", - "flaky": true + "flaky": false }, { "name": "Mac_ios integration_ui_ios_screenshot", "repo": "flutter", "task_name": "mac_ios_integration_ui_ios_screenshot", - "flaky": true + "flaky": false }, { "name": "Mac_ios integration_ui_ios_textfield", "repo": "flutter", "task_name": "mac_ios_integration_ui_ios_textfield", - "flaky": true + "flaky": false }, { "name": "Mac_ios ios_app_with_extensions_test", "repo": "flutter", "task_name": "mac_ios_ios_app_with_extensions_test", - "flaky": true + "flaky": false }, { "name": "Mac_ios ios_content_validation_test", @@ -928,31 +928,31 @@ "name": "Mac_ios ios_defines_test", "repo": "flutter", "task_name": "mac_ios_ios_defines_test", - "flaky": true + "flaky": false }, { "name": "Mac_ios ios_platform_view_tests", "repo": "flutter", "task_name": "mac_ios_ios_platform_view_tests", - "flaky": true + "flaky": false }, { "name": "Mac_ios large_image_changer_perf_ios", "repo": "flutter", "task_name": "mac_ios_large_image_changer_perf_ios", - "flaky": true + "flaky": false }, { "name": "Mac_ios macos_chrome_dev_mode", "repo": "flutter", "task_name": "mac_ios_macos_chrome_dev_mode", - "flaky": true + "flaky": false }, { "name": "Mac_ios microbenchmarks_ios", "repo": "flutter", "task_name": "mac_ios_microbenchmarks_ios", - "flaky": true + "flaky": false }, { "name": "Mac_ios new_gallery_ios__transition_perf", @@ -964,25 +964,25 @@ "name": "Mac_ios platform_channel_sample_test_ios", "repo": "flutter", "task_name": "mac_ios_platform_channel_sample_test_ios", - "flaky": true + "flaky": false }, { "name": "Mac_ios platform_channel_sample_test_swift", "repo": "flutter", "task_name": "mac_ios_platform_channel_sample_test_swift", - "flaky": true + "flaky": false }, { "name": "Mac_ios platform_interaction_test_ios", "repo": "flutter", "task_name": "mac_ios_platform_interaction_test_ios", - "flaky": true + "flaky": false }, { "name": "Mac_ios platform_view_ios__start_up", "repo": "flutter", "task_name": "mac_ios_platform_view_ios__start_up", - "flaky": true + "flaky": false }, { "name": "Mac_ios platform_views_scroll_perf_ios__timeline_summary", @@ -994,25 +994,25 @@ "name": "Mac_ios post_backdrop_filter_perf_ios__timeline_summary", "repo": "flutter", "task_name": "mac_ios_post_backdrop_filter_perf_ios__timeline_summary", - "flaky": true + "flaky": false }, { "name": "Mac_ios simple_animation_perf_ios", "repo": "flutter", "task_name": "mac_ios_simple_animation_perf_ios", - "flaky": true + "flaky": false }, { "name": "Mac_ios smoke_catalina_hot_mode_dev_cycle_ios__benchmark", "repo": "flutter", "task_name": "mac_ios_smoke_catalina_hot_mode_dev_cycle_ios__benchmark", - "flaky": true + "flaky": false }, { "name": "Mac_ios tiles_scroll_perf_ios__timeline_summary", "repo": "flutter", "task_name": "mac_ios_tiles_scroll_perf_ios__timeline_summary", - "flaky": true + "flaky": false }, { "name": "Windows build_aar_module_test",