Skip to content

Commit 474de50

Browse files
mralephmraleph
committed
Add source_gen based benchmark wrapper generator
1 parent 68df0e6 commit 474de50

8 files changed

Lines changed: 330 additions & 2 deletions

‎bin/benchmark_harness.dart‎

Lines changed: 144 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,145 @@
1-
void main() {
2-
print('Running benchmarks!');
1+
// @dart=2.9
2+
//
3+
// CLI for running lightweight microbenchmarks using Flutter tooling.
4+
//
5+
// Usage:
6+
//
7+
// flutter pub run benchmark_harness
8+
//
9+
10+
import 'dart:convert';
11+
import 'dart:io';
12+
13+
import 'package:benchmark_harness/benchmark_harness.dart';
14+
import 'package:dcli/dcli.dart';
15+
import 'package:path/path.dart' as p;
16+
17+
void main() async {
18+
// Ansi support detection does not work when running from `pub run`
19+
// force it to be always on for now.
20+
Ansi.isSupported = true;
21+
22+
// Generate benchmark wrapper scripts.
23+
print(red('Generating benchmark wrappers'));
24+
'flutter pub run build_runner build'.start(progress: Progress.devNull());
25+
26+
// Run all generated benchmarks.
27+
final resultsByFile = <String, Map<String, BenchmarkResult>>{};
28+
for (var file in find('*.benchmark.dart').toList().map(p.relative)) {
29+
resultsByFile[file] = await runBenchmarksIn(file);
30+
}
31+
32+
// Report results.
33+
print('');
34+
print('-' * 80);
35+
print('');
36+
resultsByFile.forEach((file, results) {
37+
print('Results for ${file}');
38+
final scores = {
39+
for (var r in results.values)
40+
r.name: r.elapsedMilliseconds / r.numIterations
41+
};
42+
final fastest =
43+
results.keys.reduce((a, b) => scores[a] < scores[b] ? a : b);
44+
45+
for (var result in results.values) {
46+
String suffix = '';
47+
if (result.name == fastest) {
48+
suffix = red('(fastest)');
49+
} else {
50+
double factor = scores[result.name] / scores[fastest];
51+
suffix = red('(${factor.toStringAsFixed(1)} times as slow)');
52+
}
53+
print('${result.name}: ${scores[result.name]} ms/iteration ${suffix}');
54+
}
55+
});
356
}
57+
58+
/// Runs all benchmarks in `.benchmark.dart` [file] one by one and collects
59+
/// their results.
60+
Future<Map<String, BenchmarkResult>> runBenchmarksIn(String file) async {
61+
final results = <String, BenchmarkResult>{};
62+
63+
final benchmarks = benchmarkListPattern
64+
.firstMatch(File(file).readAsStringSync())
65+
.namedGroup('list')
66+
.split(',');
67+
print(red('Found ${benchmarks.length} benchmarks in ${file}'
68+
'($benchmarks)'));
69+
for (var name in benchmarks) {
70+
results[name] = await runBenchmark(file, name);
71+
}
72+
return results;
73+
}
74+
75+
/// Runs benchmark with the given [name] defined in the given [file] and
76+
/// collects its result.
77+
Future<BenchmarkResult> runBenchmark(String file, String name) async {
78+
print(red(' measuring ${name}'));
79+
final process = await Process.start('flutter', [
80+
'run',
81+
'--release',
82+
'--machine',
83+
'--dart-define',
84+
'targetBenchmark=$name',
85+
'-t',
86+
file,
87+
]);
88+
89+
BenchmarkResult result;
90+
String appId;
91+
92+
process.stderr
93+
.transform(utf8.decoder)
94+
.transform(const LineSplitter())
95+
.listen((line) {});
96+
97+
// Process JSON-RPC events from the flutter run command.
98+
process.stdout
99+
.transform(utf8.decoder)
100+
.transform(const LineSplitter())
101+
.listen((line) {
102+
Map<String, dynamic> event;
103+
if (line.startsWith('[') && line.endsWith(']')) {
104+
event = jsonDecode(line)[0] as Map<String, dynamic>;
105+
} else {
106+
final m = benchmarkHarnessMessagePattern.firstMatch(line);
107+
if (m != null) {
108+
event = jsonDecode(m.namedGroup('event')) as Map<String, dynamic>;
109+
}
110+
}
111+
if (event == null) {
112+
return;
113+
}
114+
115+
switch (event['event'] as String) {
116+
case 'app.started':
117+
appId = event['params']['appId'] as String;
118+
break;
119+
case 'benchmark.running':
120+
print(red(' benchmark is running'));
121+
break;
122+
case 'benchmark.done':
123+
print(red(' done'));
124+
process.stdin.writeln(jsonEncode([
125+
{
126+
'id': 0,
127+
'method': 'app.stop',
128+
'params': {'appId': appId}
129+
},
130+
]));
131+
break;
132+
case 'benchmark.result':
133+
result =
134+
BenchmarkResult.fromJson(event['params'] as Map<String, dynamic>);
135+
break;
136+
}
137+
});
138+
await process.exitCode;
139+
return result;
140+
}
141+
142+
final benchmarkListPattern =
143+
RegExp(r'^// BENCHMARKS: (?<list>.*)$', multiLine: true);
144+
final benchmarkHarnessMessagePattern =
145+
RegExp(r'benchmark_harness\[(?<event>.*)\]$');

‎build.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
builders:
2+
benchmark:
3+
import: "package:benchmark_harness/builder.dart"
4+
builder_factories: ["benchmarkLibraryBuilder"]
5+
build_extensions: {".dart": [".benchmark.dart"]}
6+
auto_apply: dependents
7+
build_to: source

‎lib/annotations.dart‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
class Benchmark {
6+
const Benchmark();
7+
}
8+
9+
/// Marks top-level function as a benchmark which can be discovered and
10+
/// run by benchmark_harness CLI tooling.
11+
const benchmark = Benchmark();

‎lib/benchmark_harness.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
library benchmark_harness;
22

3+
export 'annotations.dart';
4+
35
part 'src/benchmark_base.dart';
46
part 'src/score_emitter.dart';

‎lib/benchmark_runner.dart‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/// Library for running benchmarks through benchmark_harness CLI.
2+
import 'dart:convert' show jsonEncode;
3+
4+
class BenchmarkResult {
5+
final String name;
6+
final int elapsedMilliseconds;
7+
final int numIterations;
8+
9+
BenchmarkResult({
10+
required this.name,
11+
required this.elapsedMilliseconds,
12+
required this.numIterations,
13+
});
14+
15+
BenchmarkResult.fromJson(Map<String, dynamic> result)
16+
: this(
17+
name: result['name'] as String,
18+
elapsedMilliseconds: result['elapsed'] as int,
19+
numIterations: result['iterations'] as int,
20+
);
21+
22+
Map<String, dynamic> toJson() => {
23+
'name': name,
24+
'elapsed': elapsedMilliseconds,
25+
'iterations': numIterations,
26+
};
27+
}
28+
29+
/// Runs the given measured [loop] function with an exponentially increasing
30+
/// parameter values until it finds one that causes [loop] to run for at
31+
/// least [thresholdMilliseconds] and returns [BenchmarkResult] describing
32+
/// that run.
33+
BenchmarkResult measure(void Function(int) loop,
34+
{required String name, int thresholdMilliseconds = 5000}) {
35+
var n = 2;
36+
final sw = Stopwatch();
37+
do {
38+
n *= 2;
39+
sw.reset();
40+
sw.start();
41+
loop(n);
42+
sw.stop();
43+
} while (sw.elapsedMilliseconds < thresholdMilliseconds);
44+
45+
return BenchmarkResult(
46+
name: name,
47+
elapsedMilliseconds: sw.elapsedMilliseconds,
48+
numIterations: n,
49+
);
50+
}
51+
52+
void runBenchmarks(Map<String, void Function(int)> benchmarks) {
53+
_event('benchmark.running');
54+
for (var entry in benchmarks.entries) {
55+
_event('benchmark.result', measure(entry.value, name: entry.key));
56+
}
57+
_event('benchmark.done');
58+
}
59+
60+
void _event(String event, [dynamic params]) {
61+
final encoded = jsonEncode({
62+
'event': event,
63+
if (params != null) 'params': params,
64+
});
65+
print('benchmark_harness[$encoded]');
66+
}

‎lib/builder.dart‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
/// Configuration for using `package:build`-compatible build systems.
6+
///
7+
/// See:
8+
/// * [build_runner](https://pub.dev/packages/build_runner)
9+
///
10+
/// This library is **not** intended to be imported by typical end-users unless
11+
/// you are creating a custom compilation pipeline. See documentation for
12+
/// details, and `build.yaml` for how these builders are configured by default.
13+
library benchmark_harness.builder;
14+
15+
import 'package:build/build.dart';
16+
import 'package:source_gen/source_gen.dart';
17+
18+
import 'src/benchmark_generator.dart';
19+
20+
Builder benchmarkLibraryBuilder(BuilderOptions options) =>
21+
LibraryBuilder(BenchmarkGenerator(), generatedExtension: '.benchmark.dart');

‎lib/src/benchmark_generator.dart‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
import 'dart:async';
5+
6+
import 'package:analyzer/dart/ast/ast.dart';
7+
import 'package:analyzer/dart/element/element.dart';
8+
import 'package:build/build.dart';
9+
import 'package:source_gen/source_gen.dart';
10+
11+
import '../annotations.dart';
12+
13+
class BenchmarkGenerator extends GeneratorForAnnotation<Benchmark> {
14+
@override
15+
FutureOr<String> generate(LibraryReader library, BuildStep buildStep) async {
16+
final wrappers = await super.generate(library, buildStep);
17+
if (wrappers.isEmpty) return wrappers;
18+
19+
final names = library.annotatedWith(typeChecker).map((v) => v.element.name);
20+
// benchmark_harness CLI uses BENCHMARKS line to extract the list of
21+
// benchmarks contained in this file.
22+
final defines = [
23+
'''
24+
// BENCHMARKS: ${names.join(',')}
25+
const _targetBenchmark =
26+
String.fromEnvironment('targetBenchmark', defaultValue: 'all');
27+
const _shouldMeasureAll = _targetBenchmark == 'all';
28+
''',
29+
for (var name in names)
30+
'''
31+
const _shouldMeasure\$$name = _shouldMeasureAll || _targetBenchmark == '$name';
32+
''',
33+
].join('\n');
34+
final benchmarks = [
35+
for (var name in names)
36+
'''
37+
if (_shouldMeasure\$$name)
38+
'$name': ${loopFunctionNameFor(name)},
39+
''',
40+
].join('\n');
41+
return '''
42+
import 'package:benchmark_harness/benchmark_runner.dart' as benchmark_runner;
43+
44+
import '${library.element.source.uri}' as lib;
45+
46+
$wrappers
47+
48+
$defines
49+
50+
void main() {
51+
benchmark_runner.runBenchmarks(const {
52+
$benchmarks
53+
});
54+
}
55+
''';
56+
}
57+
58+
static String loopFunctionNameFor(String name) {
59+
return '_\$measuredLoop\$$name';
60+
}
61+
62+
@override
63+
String generateForAnnotatedElement(
64+
Element element, ConstantReader annotation, BuildStep buildStep) {
65+
return '''
66+
void ${loopFunctionNameFor(element.name)}(int numIterations) {
67+
while (numIterations-- > 0) {
68+
lib.${element.name}();
69+
}
70+
}
71+
''';
72+
}
73+
}

‎pubspec.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ homepage: https://github.com/dart-lang/benchmark_harness
77
environment:
88
sdk: '>=2.12.0-0 <3.0.0'
99

10+
dependencies:
11+
analyzer: ^0.41.1
12+
build: ^1.6.0
13+
source_gen: ^0.9.10+1
14+
dcli: ^0.50.0-nullsaftey.0
15+
1016
dev_dependencies:
1117
build_runner: ^1.1.0
1218
build_web_compilers: '>=1.0.0 <3.0.0'

0 commit comments

Comments
 (0)