Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add flutter startup benchmark (#111658)
Add flutter startup benchmark for Linux, Windows and MacOs. Via guide on #111461 (comment)
- Loading branch information
Showing
6 changed files
with
233 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
// Copyright 2014 The Flutter Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style license that can be | ||
// found in the LICENSE file. | ||
|
||
import 'package:flutter_devicelab/framework/framework.dart'; | ||
import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; | ||
|
||
Future<void> main() async { | ||
await task(flutterToolStartupBenchmarkTask); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
// Copyright 2014 The Flutter Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style license that can be | ||
// found in the LICENSE file. | ||
|
||
import 'package:flutter_devicelab/framework/framework.dart'; | ||
import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; | ||
|
||
Future<void> main() async { | ||
await task(flutterToolStartupBenchmarkTask); | ||
} |
10 changes: 10 additions & 0 deletions
10
dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
// Copyright 2014 The Flutter Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style license that can be | ||
// found in the LICENSE file. | ||
|
||
import 'package:flutter_devicelab/framework/framework.dart'; | ||
import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart'; | ||
|
||
Future<void> main() async { | ||
await task(flutterToolStartupBenchmarkTask); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,170 @@ | ||
// Copyright 2014 The Flutter Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style license that can be | ||
// found in the LICENSE file. | ||
|
||
import 'dart:io'; | ||
|
||
import 'package:path/path.dart' as path; | ||
|
||
import '../framework/task_result.dart'; | ||
import '../framework/utils.dart'; | ||
|
||
/// Run each benchmark this many times and compute average, min, max. | ||
const int _kRunsPerBenchmark = 10; | ||
|
||
Future<TaskResult> flutterToolStartupBenchmarkTask() async { | ||
final Directory projectParentDirectory = | ||
Directory.systemTemp.createTempSync('flutter_tool_startup_benchmark'); | ||
final Directory projectDirectory = | ||
dir(path.join(projectParentDirectory.path, 'benchmark')); | ||
await inDirectory<void>(flutterDirectory, () async { | ||
await flutter('update-packages'); | ||
await flutter('create', options: <String>[projectDirectory.path]); | ||
// Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool | ||
rmTree(dir(path.join(projectDirectory.path, 'test'))); | ||
}); | ||
|
||
final Map<String, dynamic> data = <String, dynamic>{ | ||
// `flutter test` in dir with no `test` folder. | ||
...(await _Benchmark( | ||
projectDirectory, | ||
'test startup', | ||
'test', | ||
).run()) | ||
.asMap('flutter_tool_startup_test'), | ||
|
||
// `flutter test -d foo_device` in dir woth no `test` folder. | ||
...(await _Benchmark( | ||
projectDirectory, | ||
'test startup with specified device', | ||
'test', | ||
options: <String>['-d', 'foo_device'], | ||
).run()) | ||
.asMap('flutter_tool_startup_test_with_specified_device'), | ||
|
||
// `flutter test -v` where no android sdk will be found (at least currently). | ||
...(await _Benchmark( | ||
projectDirectory, | ||
'test startup no android sdk', | ||
'test', | ||
options: <String>['-v'], | ||
environment: <String, String>{ | ||
'ANDROID_HOME': 'dummy value', | ||
'ANDROID_SDK_ROOT': 'dummy value', | ||
'PATH': pathWithoutWhereHits(<String>['adb', 'aapt']), | ||
}, | ||
).run()) | ||
.asMap('flutter_tool_startup_test_no_android_sdk'), | ||
|
||
// `flutter -h`. | ||
...(await _Benchmark( | ||
projectDirectory, | ||
'help startup', | ||
'-h', | ||
).run()) | ||
.asMap('flutter_tool_startup_help'), | ||
}; | ||
|
||
// Cleanup. | ||
rmTree(projectParentDirectory); | ||
|
||
return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); | ||
} | ||
|
||
String pathWithoutWhereHits(List<String> whats) { | ||
final String pathEnvironment = Platform.environment['PATH'] ?? ''; | ||
List<String> paths; | ||
if (Platform.isWindows) { | ||
paths = pathEnvironment.split(';'); | ||
} else { | ||
paths = pathEnvironment.split(':'); | ||
} | ||
// This isn't great but will probably work for our purposes. | ||
final List<String> extensions = <String>['', '.exe', '.bat', '.com']; | ||
|
||
final List<String> notFound = <String>[]; | ||
for (final String path in paths) { | ||
bool found = false; | ||
for (final String extension in extensions) { | ||
for (final String what in whats) { | ||
final File f = File('$path${Platform.pathSeparator}$what$extension'); | ||
if (f.existsSync()) { | ||
found = true; | ||
break; | ||
} | ||
} | ||
if (found) { | ||
break; | ||
} | ||
} | ||
if (!found) { | ||
notFound.add(path); | ||
} | ||
} | ||
|
||
if (Platform.isWindows) { | ||
return notFound.join(';'); | ||
} else { | ||
return notFound.join(':'); | ||
} | ||
} | ||
|
||
class _BenchmarkResult { | ||
const _BenchmarkResult(this.mean, this.min, this.max); | ||
|
||
final int mean; // Milliseconds | ||
|
||
final int min; // Milliseconds | ||
|
||
final int max; // Milliseconds | ||
|
||
Map<String, dynamic> asMap(String name) { | ||
return <String, dynamic>{ | ||
name: mean, | ||
'${name}_minimum': min, | ||
'${name}_maximum': max, | ||
}; | ||
} | ||
} | ||
|
||
class _Benchmark { | ||
_Benchmark(this.directory, this.title, this.command, | ||
{this.options = const <String>[], this.environment}); | ||
|
||
final Directory directory; | ||
|
||
final String title; | ||
|
||
final String command; | ||
|
||
final List<String> options; | ||
|
||
final Map<String, String>? environment; | ||
|
||
Future<int> execute(int iteration, int targetIterations) async { | ||
section('Benchmark $title - ${iteration + 1} / $targetIterations'); | ||
final Stopwatch stopwatch = Stopwatch(); | ||
await inDirectory<void>(directory, () async { | ||
stopwatch.start(); | ||
// canFail is set to true, as e.g. `flutter test` in a dir with no `test` | ||
// directory sets a non-zero return value. | ||
await flutter(command, | ||
options: options, canFail: true, environment: environment); | ||
stopwatch.stop(); | ||
}); | ||
return stopwatch.elapsedMilliseconds; | ||
} | ||
|
||
/// Runs `benchmark` several times and reports the results. | ||
Future<_BenchmarkResult> run() async { | ||
final List<int> results = <int>[]; | ||
int sum = 0; | ||
for (int i = 0; i < _kRunsPerBenchmark; i++) { | ||
final int thisRuntime = await execute(i, _kRunsPerBenchmark); | ||
results.add(thisRuntime); | ||
sum += thisRuntime; | ||
} | ||
results.sort(); | ||
return _BenchmarkResult(sum ~/ results.length, results.first, results.last); | ||
} | ||
} |