forked from dart-archive/benchmark_harness
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add source_gen based benchmark wrapper generator
- Loading branch information
Showing
8 changed files
with
330 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,145 @@ | ||
void main() { | ||
print('Running benchmarks!'); | ||
// @dart=2.9 | ||
// | ||
// CLI for running lightweight microbenchmarks using Flutter tooling. | ||
// | ||
// Usage: | ||
// | ||
// flutter pub run benchmark_harness | ||
// | ||
|
||
import 'dart:convert'; | ||
import 'dart:io'; | ||
|
||
import 'package:benchmark_harness/benchmark_harness.dart'; | ||
import 'package:dcli/dcli.dart'; | ||
import 'package:path/path.dart' as p; | ||
|
||
void main() async { | ||
// Ansi support detection does not work when running from `pub run` | ||
// force it to be always on for now. | ||
Ansi.isSupported = true; | ||
|
||
// Generate benchmark wrapper scripts. | ||
print(red('Generating benchmark wrappers')); | ||
'flutter pub run build_runner build'.start(progress: Progress.devNull()); | ||
|
||
// Run all generated benchmarks. | ||
final resultsByFile = <String, Map<String, BenchmarkResult>>{}; | ||
for (var file in find('*.benchmark.dart').toList().map(p.relative)) { | ||
resultsByFile[file] = await runBenchmarksIn(file); | ||
} | ||
|
||
// Report results. | ||
print(''); | ||
print('-' * 80); | ||
print(''); | ||
resultsByFile.forEach((file, results) { | ||
print('Results for ${file}'); | ||
final scores = { | ||
for (var r in results.values) | ||
r.name: r.elapsedMilliseconds / r.numIterations | ||
}; | ||
final fastest = | ||
results.keys.reduce((a, b) => scores[a] < scores[b] ? a : b); | ||
|
||
for (var result in results.values) { | ||
String suffix = ''; | ||
if (result.name == fastest) { | ||
suffix = red('(fastest)'); | ||
} else { | ||
double factor = scores[result.name] / scores[fastest]; | ||
suffix = red('(${factor.toStringAsFixed(1)} times as slow)'); | ||
} | ||
print('${result.name}: ${scores[result.name]} ms/iteration ${suffix}'); | ||
} | ||
}); | ||
} | ||
|
||
/// Runs all benchmarks in `.benchmark.dart` [file] one by one and collects | ||
/// their results. | ||
Future<Map<String, BenchmarkResult>> runBenchmarksIn(String file) async { | ||
final results = <String, BenchmarkResult>{}; | ||
|
||
final benchmarks = benchmarkListPattern | ||
.firstMatch(File(file).readAsStringSync()) | ||
.namedGroup('list') | ||
.split(','); | ||
print(red('Found ${benchmarks.length} benchmarks in ${file}' | ||
'($benchmarks)')); | ||
for (var name in benchmarks) { | ||
results[name] = await runBenchmark(file, name); | ||
} | ||
return results; | ||
} | ||
|
||
/// Runs benchmark with the given [name] defined in the given [file] and | ||
/// collects its result. | ||
Future<BenchmarkResult> runBenchmark(String file, String name) async { | ||
print(red(' measuring ${name}')); | ||
final process = await Process.start('flutter', [ | ||
'run', | ||
'--release', | ||
'--machine', | ||
'--dart-define', | ||
'targetBenchmark=$name', | ||
'-t', | ||
file, | ||
]); | ||
|
||
BenchmarkResult result; | ||
String appId; | ||
|
||
process.stderr | ||
.transform(utf8.decoder) | ||
.transform(const LineSplitter()) | ||
.listen((line) {}); | ||
|
||
// Process JSON-RPC events from the flutter run command. | ||
process.stdout | ||
.transform(utf8.decoder) | ||
.transform(const LineSplitter()) | ||
.listen((line) { | ||
Map<String, dynamic> event; | ||
if (line.startsWith('[') && line.endsWith(']')) { | ||
event = jsonDecode(line)[0] as Map<String, dynamic>; | ||
} else { | ||
final m = benchmarkHarnessMessagePattern.firstMatch(line); | ||
if (m != null) { | ||
event = jsonDecode(m.namedGroup('event')) as Map<String, dynamic>; | ||
} | ||
} | ||
if (event == null) { | ||
return; | ||
} | ||
|
||
switch (event['event'] as String) { | ||
case 'app.started': | ||
appId = event['params']['appId'] as String; | ||
break; | ||
case 'benchmark.running': | ||
print(red(' benchmark is running')); | ||
break; | ||
case 'benchmark.done': | ||
print(red(' done')); | ||
process.stdin.writeln(jsonEncode([ | ||
{ | ||
'id': 0, | ||
'method': 'app.stop', | ||
'params': {'appId': appId} | ||
}, | ||
])); | ||
break; | ||
case 'benchmark.result': | ||
result = | ||
BenchmarkResult.fromJson(event['params'] as Map<String, dynamic>); | ||
break; | ||
} | ||
}); | ||
await process.exitCode; | ||
return result; | ||
} | ||
|
||
final benchmarkListPattern = | ||
RegExp(r'^// BENCHMARKS: (?<list>.*)$', multiLine: true); | ||
final benchmarkHarnessMessagePattern = | ||
RegExp(r'benchmark_harness\[(?<event>.*)\]$'); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
builders: | ||
benchmark: | ||
import: "package:benchmark_harness/builder.dart" | ||
builder_factories: ["benchmarkLibraryBuilder"] | ||
build_extensions: {".dart": [".benchmark.dart"]} | ||
auto_apply: dependents | ||
build_to: source |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file | ||
// for details. All rights reserved. Use of this source code is governed by a | ||
// BSD-style license that can be found in the LICENSE file. | ||
|
||
class Benchmark { | ||
const Benchmark(); | ||
} | ||
|
||
/// Marks top-level function as a benchmark which can be discovered and | ||
/// run by benchmark_harness CLI tooling. | ||
const benchmark = Benchmark(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,6 @@ | ||
library benchmark_harness; | ||
|
||
export 'annotations.dart'; | ||
|
||
part 'src/benchmark_base.dart'; | ||
part 'src/score_emitter.dart'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
/// Library for running benchmarks through benchmark_harness CLI. | ||
import 'dart:convert' show jsonEncode; | ||
|
||
class BenchmarkResult { | ||
final String name; | ||
final int elapsedMilliseconds; | ||
final int numIterations; | ||
|
||
BenchmarkResult({ | ||
required this.name, | ||
required this.elapsedMilliseconds, | ||
required this.numIterations, | ||
}); | ||
|
||
BenchmarkResult.fromJson(Map<String, dynamic> result) | ||
: this( | ||
name: result['name'] as String, | ||
elapsedMilliseconds: result['elapsed'] as int, | ||
numIterations: result['iterations'] as int, | ||
); | ||
|
||
Map<String, dynamic> toJson() => { | ||
'name': name, | ||
'elapsed': elapsedMilliseconds, | ||
'iterations': numIterations, | ||
}; | ||
} | ||
|
||
/// Runs the given measured [loop] function with an exponentially increasing | ||
/// parameter values until it finds one that causes [loop] to run for at | ||
/// least [thresholdMilliseconds] and returns [BenchmarkResult] describing | ||
/// that run. | ||
BenchmarkResult measure(void Function(int) loop, | ||
{required String name, int thresholdMilliseconds = 5000}) { | ||
var n = 2; | ||
final sw = Stopwatch(); | ||
do { | ||
n *= 2; | ||
sw.reset(); | ||
sw.start(); | ||
loop(n); | ||
sw.stop(); | ||
} while (sw.elapsedMilliseconds < thresholdMilliseconds); | ||
|
||
return BenchmarkResult( | ||
name: name, | ||
elapsedMilliseconds: sw.elapsedMilliseconds, | ||
numIterations: n, | ||
); | ||
} | ||
|
||
void runBenchmarks(Map<String, void Function(int)> benchmarks) { | ||
_event('benchmark.running'); | ||
for (var entry in benchmarks.entries) { | ||
_event('benchmark.result', measure(entry.value, name: entry.key)); | ||
} | ||
_event('benchmark.done'); | ||
} | ||
|
||
void _event(String event, [dynamic params]) { | ||
final encoded = jsonEncode({ | ||
'event': event, | ||
if (params != null) 'params': params, | ||
}); | ||
print('benchmark_harness[$encoded]'); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file | ||
// for details. All rights reserved. Use of this source code is governed by a | ||
// BSD-style license that can be found in the LICENSE file. | ||
|
||
/// Configuration for using `package:build`-compatible build systems. | ||
/// | ||
/// See: | ||
/// * [build_runner](https://pub.dev/packages/build_runner) | ||
/// | ||
/// This library is **not** intended to be imported by typical end-users unless | ||
/// you are creating a custom compilation pipeline. See documentation for | ||
/// details, and `build.yaml` for how these builders are configured by default. | ||
library benchmark_harness.builder; | ||
|
||
import 'package:build/build.dart'; | ||
import 'package:source_gen/source_gen.dart'; | ||
|
||
import 'src/benchmark_generator.dart'; | ||
|
||
Builder benchmarkLibraryBuilder(BuilderOptions options) => | ||
LibraryBuilder(BenchmarkGenerator(), generatedExtension: '.benchmark.dart'); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file | ||
// for details. All rights reserved. Use of this source code is governed by a | ||
// BSD-style license that can be found in the LICENSE file. | ||
import 'dart:async'; | ||
|
||
import 'package:analyzer/dart/ast/ast.dart'; | ||
import 'package:analyzer/dart/element/element.dart'; | ||
import 'package:build/build.dart'; | ||
import 'package:source_gen/source_gen.dart'; | ||
|
||
import '../annotations.dart'; | ||
|
||
class BenchmarkGenerator extends GeneratorForAnnotation<Benchmark> { | ||
@override | ||
FutureOr<String> generate(LibraryReader library, BuildStep buildStep) async { | ||
final wrappers = await super.generate(library, buildStep); | ||
if (wrappers.isEmpty) return wrappers; | ||
|
||
final names = library.annotatedWith(typeChecker).map((v) => v.element.name); | ||
// benchmark_harness CLI uses BENCHMARKS line to extract the list of | ||
// benchmarks contained in this file. | ||
final defines = [ | ||
''' | ||
// BENCHMARKS: ${names.join(',')} | ||
const _targetBenchmark = | ||
String.fromEnvironment('targetBenchmark', defaultValue: 'all'); | ||
const _shouldMeasureAll = _targetBenchmark == 'all'; | ||
''', | ||
for (var name in names) | ||
''' | ||
const _shouldMeasure\$$name = _shouldMeasureAll || _targetBenchmark == '$name'; | ||
''', | ||
].join('\n'); | ||
final benchmarks = [ | ||
for (var name in names) | ||
''' | ||
if (_shouldMeasure\$$name) | ||
'$name': ${loopFunctionNameFor(name)}, | ||
''', | ||
].join('\n'); | ||
return ''' | ||
import 'package:benchmark_harness/benchmark_runner.dart' as benchmark_runner; | ||
import '${library.element.source.uri}' as lib; | ||
$wrappers | ||
$defines | ||
void main() { | ||
benchmark_runner.runBenchmarks(const { | ||
$benchmarks | ||
}); | ||
} | ||
'''; | ||
} | ||
|
||
static String loopFunctionNameFor(String name) { | ||
return '_\$measuredLoop\$$name'; | ||
} | ||
|
||
@override | ||
String generateForAnnotatedElement( | ||
Element element, ConstantReader annotation, BuildStep buildStep) { | ||
return ''' | ||
void ${loopFunctionNameFor(element.name)}(int numIterations) { | ||
while (numIterations-- > 0) { | ||
lib.${element.name}(); | ||
} | ||
} | ||
'''; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters