From b7b8b759bc3ab7a80d2576d52f7b05bc1e6e23bd Mon Sep 17 00:00:00 2001 From: jensjoha Date: Fri, 16 Sep 2022 08:41:30 +0200 Subject: [PATCH] Add flutter startup benchmark (#111658) Add flutter startup benchmark for Linux, Windows and MacOs. Via guide on https://github.com/flutter/flutter/pull/111461#issuecomment-1247113905 --- .ci.yaml | 30 ++++ TESTOWNERS | 3 + .../tasks/flutter_tool_startup__linux.dart | 10 ++ .../tasks/flutter_tool_startup__macos.dart | 10 ++ .../tasks/flutter_tool_startup__windows.dart | 10 ++ .../lib/tasks/flutter_tool_startup.dart | 170 ++++++++++++++++++ 6 files changed, 233 insertions(+) create mode 100644 dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart create mode 100644 dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart create mode 100644 dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart create mode 100644 dev/devicelab/lib/tasks/flutter_tool_startup.dart diff --git a/.ci.yaml b/.ci.yaml index 0adff14553e0..59461e46b574 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -4445,3 +4445,33 @@ targets: tags: > ["devicelab", "hostonly"] task_name: windows_startup_test + + - name: Windows flutter_tool_startup__windows + bringup: true + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + properties: + tags: > + ["devicelab", "hostonly"] + task_name: flutter_tool_startup__windows + + - name: Linux flutter_tool_startup__linux + bringup: true + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + properties: + tags: > + ["devicelab", "hostonly"] + task_name: flutter_tool_startup__linux + + - name: Mac flutter_tool_startup__macos + bringup: true + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + properties: + tags: > + ["devicelab", "hostonly"] + task_name: flutter_tool_startup__macos diff --git a/TESTOWNERS b/TESTOWNERS index 692684183172..f30f239cb362 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -220,6 +220,9 @@ /dev/devicelab/bin/tasks/flutter_gallery_win_desktop__compile.dart @yaakovschectman @flutter/desktop /dev/devicelab/bin/tasks/flutter_gallery_win_desktop__start_up.dart @yaakovschectman @flutter/desktop /dev/devicelab/bin/tasks/flutter_view_macos__start_up.dart @a-wallen @flutter/desktop +/dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart @jensjoha @flutter/tool +/dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart @jensjoha @flutter/tool +/dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart @jensjoha @flutter/tool /dev/devicelab/bin/tasks/flutter_view_win_desktop__start_up.dart @yaakovschectman @flutter/desktop /dev/devicelab/bin/tasks/gradle_desugar_classes_test.dart @zanderso @flutter/tool /dev/devicelab/bin/tasks/gradle_non_android_plugin_test.dart @stuartmorgan @flutter/plugin diff --git a/dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart b/dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart new file mode 100644 index 000000000000..af38f6e1177a --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; + +Future main() async { + await task(flutterToolStartupBenchmarkTask); +} diff --git a/dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart b/dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart new file mode 100644 index 000000000000..af38f6e1177a --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; + +Future main() async { + await task(flutterToolStartupBenchmarkTask); +} diff --git a/dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart b/dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart new file mode 100644 index 000000000000..af38f6e1177a --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; + +Future main() async { + await task(flutterToolStartupBenchmarkTask); +} diff --git a/dev/devicelab/lib/tasks/flutter_tool_startup.dart b/dev/devicelab/lib/tasks/flutter_tool_startup.dart new file mode 100644 index 000000000000..771dfe9c6004 --- /dev/null +++ b/dev/devicelab/lib/tasks/flutter_tool_startup.dart @@ -0,0 +1,170 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import '../framework/task_result.dart'; +import '../framework/utils.dart'; + +/// Run each benchmark this many times and compute average, min, max. +const int _kRunsPerBenchmark = 10; + +Future flutterToolStartupBenchmarkTask() async { + final Directory projectParentDirectory = + Directory.systemTemp.createTempSync('flutter_tool_startup_benchmark'); + final Directory projectDirectory = + dir(path.join(projectParentDirectory.path, 'benchmark')); + await inDirectory(flutterDirectory, () async { + await flutter('update-packages'); + await flutter('create', options: [projectDirectory.path]); + // Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool + rmTree(dir(path.join(projectDirectory.path, 'test'))); + }); + + final Map data = { + // `flutter test` in dir with no `test` folder. + ...(await _Benchmark( + projectDirectory, + 'test startup', + 'test', + ).run()) + .asMap('flutter_tool_startup_test'), + + // `flutter test -d foo_device` in dir woth no `test` folder. + ...(await _Benchmark( + projectDirectory, + 'test startup with specified device', + 'test', + options: ['-d', 'foo_device'], + ).run()) + .asMap('flutter_tool_startup_test_with_specified_device'), + + // `flutter test -v` where no android sdk will be found (at least currently). + ...(await _Benchmark( + projectDirectory, + 'test startup no android sdk', + 'test', + options: ['-v'], + environment: { + 'ANDROID_HOME': 'dummy value', + 'ANDROID_SDK_ROOT': 'dummy value', + 'PATH': pathWithoutWhereHits(['adb', 'aapt']), + }, + ).run()) + .asMap('flutter_tool_startup_test_no_android_sdk'), + + // `flutter -h`. + ...(await _Benchmark( + projectDirectory, + 'help startup', + '-h', + ).run()) + .asMap('flutter_tool_startup_help'), + }; + + // Cleanup. + rmTree(projectParentDirectory); + + return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); +} + +String pathWithoutWhereHits(List whats) { + final String pathEnvironment = Platform.environment['PATH'] ?? ''; + List paths; + if (Platform.isWindows) { + paths = pathEnvironment.split(';'); + } else { + paths = pathEnvironment.split(':'); + } + // This isn't great but will probably work for our purposes. + final List extensions = ['', '.exe', '.bat', '.com']; + + final List notFound = []; + for (final String path in paths) { + bool found = false; + for (final String extension in extensions) { + for (final String what in whats) { + final File f = File('$path${Platform.pathSeparator}$what$extension'); + if (f.existsSync()) { + found = true; + break; + } + } + if (found) { + break; + } + } + if (!found) { + notFound.add(path); + } + } + + if (Platform.isWindows) { + return notFound.join(';'); + } else { + return notFound.join(':'); + } +} + +class _BenchmarkResult { + const _BenchmarkResult(this.mean, this.min, this.max); + + final int mean; // Milliseconds + + final int min; // Milliseconds + + final int max; // Milliseconds + + Map asMap(String name) { + return { + name: mean, + '${name}_minimum': min, + '${name}_maximum': max, + }; + } +} + +class _Benchmark { + _Benchmark(this.directory, this.title, this.command, + {this.options = const [], this.environment}); + + final Directory directory; + + final String title; + + final String command; + + final List options; + + final Map? environment; + + Future execute(int iteration, int targetIterations) async { + section('Benchmark $title - ${iteration + 1} / $targetIterations'); + final Stopwatch stopwatch = Stopwatch(); + await inDirectory(directory, () async { + stopwatch.start(); + // canFail is set to true, as e.g. `flutter test` in a dir with no `test` + // directory sets a non-zero return value. + await flutter(command, + options: options, canFail: true, environment: environment); + stopwatch.stop(); + }); + return stopwatch.elapsedMilliseconds; + } + + /// Runs `benchmark` several times and reports the results. + Future<_BenchmarkResult> run() async { + final List results = []; + int sum = 0; + for (int i = 0; i < _kRunsPerBenchmark; i++) { + final int thisRuntime = await execute(i, _kRunsPerBenchmark); + results.add(thisRuntime); + sum += thisRuntime; + } + results.sort(); + return _BenchmarkResult(sum ~/ results.length, results.first, results.last); + } +}