Skip to content

Commit

Permalink
Add source_gen based benchmark wrapper generator
Browse files Browse the repository at this point in the history
  • Loading branch information
mraleph committed Jan 14, 2021
1 parent 68df0e6 commit 474de50
Show file tree
Hide file tree
Showing 8 changed files with 330 additions and 2 deletions.
146 changes: 144 additions & 2 deletions bin/benchmark_harness.dart
Original file line number Diff line number Diff line change
@@ -1,3 +1,145 @@
void main() {
print('Running benchmarks!');
// @dart=2.9
//
// CLI for running lightweight microbenchmarks using Flutter tooling.
//
// Usage:
//
// flutter pub run benchmark_harness
//

import 'dart:convert';
import 'dart:io';

import 'package:benchmark_harness/benchmark_harness.dart';
import 'package:dcli/dcli.dart';
import 'package:path/path.dart' as p;

void main() async {
// Ansi support detection does not work when running from `pub run`
// force it to be always on for now.
Ansi.isSupported = true;

// Generate benchmark wrapper scripts.
print(red('Generating benchmark wrappers'));
'flutter pub run build_runner build'.start(progress: Progress.devNull());

// Run all generated benchmarks.
final resultsByFile = <String, Map<String, BenchmarkResult>>{};
for (var file in find('*.benchmark.dart').toList().map(p.relative)) {
resultsByFile[file] = await runBenchmarksIn(file);
}

// Report results.
print('');
print('-' * 80);
print('');
resultsByFile.forEach((file, results) {
print('Results for ${file}');
final scores = {
for (var r in results.values)
r.name: r.elapsedMilliseconds / r.numIterations
};
final fastest =
results.keys.reduce((a, b) => scores[a] < scores[b] ? a : b);

for (var result in results.values) {
String suffix = '';
if (result.name == fastest) {
suffix = red('(fastest)');
} else {
double factor = scores[result.name] / scores[fastest];
suffix = red('(${factor.toStringAsFixed(1)} times as slow)');
}
print('${result.name}: ${scores[result.name]} ms/iteration ${suffix}');
}
});
}

/// Runs all benchmarks in `.benchmark.dart` [file] one by one and collects
/// their results.
Future<Map<String, BenchmarkResult>> runBenchmarksIn(String file) async {
final results = <String, BenchmarkResult>{};

final benchmarks = benchmarkListPattern
.firstMatch(File(file).readAsStringSync())
.namedGroup('list')
.split(',');
print(red('Found ${benchmarks.length} benchmarks in ${file}'
'($benchmarks)'));
for (var name in benchmarks) {
results[name] = await runBenchmark(file, name);
}
return results;
}

/// Runs benchmark with the given [name] defined in the given [file] and
/// collects its result.
Future<BenchmarkResult> runBenchmark(String file, String name) async {
print(red(' measuring ${name}'));
final process = await Process.start('flutter', [
'run',
'--release',
'--machine',
'--dart-define',
'targetBenchmark=$name',
'-t',
file,
]);

BenchmarkResult result;
String appId;

process.stderr
.transform(utf8.decoder)
.transform(const LineSplitter())
.listen((line) {});

// Process JSON-RPC events from the flutter run command.
process.stdout
.transform(utf8.decoder)
.transform(const LineSplitter())
.listen((line) {
Map<String, dynamic> event;
if (line.startsWith('[') && line.endsWith(']')) {
event = jsonDecode(line)[0] as Map<String, dynamic>;
} else {
final m = benchmarkHarnessMessagePattern.firstMatch(line);
if (m != null) {
event = jsonDecode(m.namedGroup('event')) as Map<String, dynamic>;
}
}
if (event == null) {
return;
}

switch (event['event'] as String) {
case 'app.started':
appId = event['params']['appId'] as String;
break;
case 'benchmark.running':
print(red(' benchmark is running'));
break;
case 'benchmark.done':
print(red(' done'));
process.stdin.writeln(jsonEncode([
{
'id': 0,
'method': 'app.stop',
'params': {'appId': appId}
},
]));
break;
case 'benchmark.result':
result =
BenchmarkResult.fromJson(event['params'] as Map<String, dynamic>);
break;
}
});
await process.exitCode;
return result;
}

final benchmarkListPattern =
RegExp(r'^// BENCHMARKS: (?<list>.*)$', multiLine: true);
final benchmarkHarnessMessagePattern =
RegExp(r'benchmark_harness\[(?<event>.*)\]$');
7 changes: 7 additions & 0 deletions build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
builders:
benchmark:
import: "package:benchmark_harness/builder.dart"
builder_factories: ["benchmarkLibraryBuilder"]
build_extensions: {".dart": [".benchmark.dart"]}
auto_apply: dependents
build_to: source
11 changes: 11 additions & 0 deletions lib/annotations.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

class Benchmark {
const Benchmark();
}

/// Marks top-level function as a benchmark which can be discovered and
/// run by benchmark_harness CLI tooling.
const benchmark = Benchmark();
2 changes: 2 additions & 0 deletions lib/benchmark_harness.dart
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
library benchmark_harness;

export 'annotations.dart';

part 'src/benchmark_base.dart';
part 'src/score_emitter.dart';
66 changes: 66 additions & 0 deletions lib/benchmark_runner.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/// Library for running benchmarks through benchmark_harness CLI.
import 'dart:convert' show jsonEncode;

class BenchmarkResult {
final String name;
final int elapsedMilliseconds;
final int numIterations;

BenchmarkResult({
required this.name,
required this.elapsedMilliseconds,
required this.numIterations,
});

BenchmarkResult.fromJson(Map<String, dynamic> result)
: this(
name: result['name'] as String,
elapsedMilliseconds: result['elapsed'] as int,
numIterations: result['iterations'] as int,
);

Map<String, dynamic> toJson() => {
'name': name,
'elapsed': elapsedMilliseconds,
'iterations': numIterations,
};
}

/// Runs the given measured [loop] function with an exponentially increasing
/// parameter values until it finds one that causes [loop] to run for at
/// least [thresholdMilliseconds] and returns [BenchmarkResult] describing
/// that run.
BenchmarkResult measure(void Function(int) loop,
{required String name, int thresholdMilliseconds = 5000}) {
var n = 2;
final sw = Stopwatch();
do {
n *= 2;
sw.reset();
sw.start();
loop(n);
sw.stop();
} while (sw.elapsedMilliseconds < thresholdMilliseconds);

return BenchmarkResult(
name: name,
elapsedMilliseconds: sw.elapsedMilliseconds,
numIterations: n,
);
}

void runBenchmarks(Map<String, void Function(int)> benchmarks) {
_event('benchmark.running');
for (var entry in benchmarks.entries) {
_event('benchmark.result', measure(entry.value, name: entry.key));
}
_event('benchmark.done');
}

void _event(String event, [dynamic params]) {
final encoded = jsonEncode({
'event': event,
if (params != null) 'params': params,
});
print('benchmark_harness[$encoded]');
}
21 changes: 21 additions & 0 deletions lib/builder.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Configuration for using `package:build`-compatible build systems.
///
/// See:
/// * [build_runner](https://pub.dev/packages/build_runner)
///
/// This library is **not** intended to be imported by typical end-users unless
/// you are creating a custom compilation pipeline. See documentation for
/// details, and `build.yaml` for how these builders are configured by default.
library benchmark_harness.builder;

import 'package:build/build.dart';
import 'package:source_gen/source_gen.dart';

import 'src/benchmark_generator.dart';

Builder benchmarkLibraryBuilder(BuilderOptions options) =>
LibraryBuilder(BenchmarkGenerator(), generatedExtension: '.benchmark.dart');
73 changes: 73 additions & 0 deletions lib/src/benchmark_generator.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';

import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/element/element.dart';
import 'package:build/build.dart';
import 'package:source_gen/source_gen.dart';

import '../annotations.dart';

class BenchmarkGenerator extends GeneratorForAnnotation<Benchmark> {
@override
FutureOr<String> generate(LibraryReader library, BuildStep buildStep) async {
final wrappers = await super.generate(library, buildStep);
if (wrappers.isEmpty) return wrappers;

final names = library.annotatedWith(typeChecker).map((v) => v.element.name);
// benchmark_harness CLI uses BENCHMARKS line to extract the list of
// benchmarks contained in this file.
final defines = [
'''
// BENCHMARKS: ${names.join(',')}
const _targetBenchmark =
String.fromEnvironment('targetBenchmark', defaultValue: 'all');
const _shouldMeasureAll = _targetBenchmark == 'all';
''',
for (var name in names)
'''
const _shouldMeasure\$$name = _shouldMeasureAll || _targetBenchmark == '$name';
''',
].join('\n');
final benchmarks = [
for (var name in names)
'''
if (_shouldMeasure\$$name)
'$name': ${loopFunctionNameFor(name)},
''',
].join('\n');
return '''
import 'package:benchmark_harness/benchmark_runner.dart' as benchmark_runner;
import '${library.element.source.uri}' as lib;
$wrappers
$defines
void main() {
benchmark_runner.runBenchmarks(const {
$benchmarks
});
}
''';
}

static String loopFunctionNameFor(String name) {
return '_\$measuredLoop\$$name';
}

@override
String generateForAnnotatedElement(
Element element, ConstantReader annotation, BuildStep buildStep) {
return '''
void ${loopFunctionNameFor(element.name)}(int numIterations) {
while (numIterations-- > 0) {
lib.${element.name}();
}
}
''';
}
}
6 changes: 6 additions & 0 deletions pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ homepage: https://github.com/dart-lang/benchmark_harness
environment:
sdk: '>=2.12.0-0 <3.0.0'

dependencies:
analyzer: ^0.41.1
build: ^1.6.0
source_gen: ^0.9.10+1
dcli: ^0.50.0-nullsaftey.0

dev_dependencies:
build_runner: ^1.1.0
build_web_compilers: '>=1.0.0 <3.0.0'
Expand Down

0 comments on commit 474de50

Please sign in to comment.