Skip to content

Commit

Permalink
feat(test): enhance logs and final test report in case of failing tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Abdelaziz GACEMI committed Dec 17, 2022
1 parent 54cd672 commit 3730d93
Show file tree
Hide file tree
Showing 4 changed files with 195 additions and 47 deletions.
3 changes: 1 addition & 2 deletions lib/src/cli/cli.dart
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import 'dart:async';

import 'package:collection/collection.dart';
import 'package:glob/glob.dart';
import 'package:lcov_parser/lcov_parser.dart';
import 'package:mason/mason.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as p;
import 'package:pubspec_parse/pubspec_parse.dart';
import 'package:stack_trace/stack_trace.dart';
import 'package:universal_io/io.dart';
import 'package:very_good_cli/src/commands/test/templates/test_runner_bundle.dart';
import 'package:very_good_test_runner/very_good_test_runner.dart';
Expand Down
100 changes: 62 additions & 38 deletions lib/src/cli/flutter_cli.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,13 +205,13 @@ class Flutter {
'--test-randomize-ordering-seed',
randomSeed
],
if (optimizePerformance) p.join('test', '.test_runner.dart')
if (optimizePerformance) p.join('test', _testRunnerFileName)
],
stdout: stdout ?? noop,
stderr: stderr ?? noop,
).whenComplete(() {
if (optimizePerformance) {
File(p.join(cwd, 'test', '.test_runner.dart')).delete().ignore();
File(p.join(cwd, 'test', _testRunnerFileName)).delete().ignore();
}
});
},
Expand Down Expand Up @@ -313,14 +313,15 @@ Future<int> _flutterTest({
final suites = <int, TestSuite>{};
final groups = <int, TestGroup>{};
final tests = <int, Test>{};
final failedTestErrorMessages = <int, String>{};
final failedTestErrorMessages = <String, List<String>>{};

var successCount = 0;
var skipCount = 0;

String computeStats() {
final passingTests = successCount.formatSuccess();
final failingTests = failedTestErrorMessages.length.formatFailure();
final failingTests =
failedTestErrorMessages.values.expand((e) => e).length.formatFailure();
final skippedTests = skipCount.formatSkipped();
final result = [passingTests, failingTests, skippedTests]
..removeWhere((element) => element.isEmpty);
Expand Down Expand Up @@ -367,41 +368,52 @@ Future<int> _flutterTest({
if (event.stackTrace.trim().isNotEmpty) {
stderr('$clearLine${event.stackTrace}');
}

final traceLocation = _getTraceLocation(stackTrace: event.stackTrace);

// When failing to recover the location from the stack trace,
// save a short description of the error
final testErrorDescription = traceLocation ??
event.error.replaceAll('\n', ' ').truncated(_lineLength);

final test = tests[event.testID]!;
final suite = suites[test.suiteID]!;
final prefix = event.isFailure ? '[FAILED]' : '[ERROR]';
failedTestErrorMessages[event.testID] = '$prefix $testErrorDescription';

final optimizationApplied = _isOptimizationApplied(suite);
final firstGroupName = _firstGroupName(test, groups);
final testPath =
_actualTestPath(optimizationApplied, suite.path!, firstGroupName);
final testName =
_actualTestName(optimizationApplied, test.name, firstGroupName);

failedTestErrorMessages[testPath] = [
...failedTestErrorMessages[testPath] ?? [],
'$prefix $testName'
];
}

if (event is TestDoneEvent) {
if (event.hidden) return;

final test = tests[event.testID]!;
final suite = suites[test.suiteID]!;
final optimizationApplied = _isOptimizationApplied(suite);
final firstGroupName = _firstGroupName(test, groups);
final testPath =
_actualTestPath(optimizationApplied, suite.path!, firstGroupName);
final testName =
_actualTestName(optimizationApplied, test.name, firstGroupName);

if (event.skipped) {
stdout(
'''$clearLine${lightYellow.wrap('${test.name} ${suite.path} (SKIPPED)')}\n''',
'''$clearLine${lightYellow.wrap('$testName $testPath (SKIPPED)')}\n''',
);
skipCount++;
} else if (event.result == TestResult.success) {
successCount++;
} else {
stderr('$clearLine${test.name} ${suite.path} (FAILED)');
stderr('$clearLine$testName $testPath (FAILED)');
}

final timeElapsed = Duration(milliseconds: event.time).formatted();
final stats = computeStats();
final testName = test.name.truncated(
final truncatedTestName = testName.truncated(
_lineLength - (timeElapsed.length + stats.length + 2),
);
stdout('''$clearLine$timeElapsed $stats: $testName''');
stdout('''$clearLine$timeElapsed $stats: $truncatedTestName''');
}

if (event is DoneTestEvent) {
Expand All @@ -422,9 +434,15 @@ Future<int> _flutterTest({
final title = styleBold.wrap('Failing Tests:');

final lines = StringBuffer('$clearLine$title\n');
for (final errorMessage in failedTestErrorMessages.values) {
lines.writeln('$clearLine - $errorMessage');
for (final testSuiteErrorMessages
in failedTestErrorMessages.entries) {
lines.writeln('$clearLine - ${testSuiteErrorMessages.key} ');

for (final errorMessage in testSuiteErrorMessages.value) {
lines.writeln('$clearLine \t- $errorMessage');
}
}

stderr(lines.toString());
}
}
Expand All @@ -448,6 +466,31 @@ Future<int> _flutterTest({
return completer.future;
}

bool _isOptimizationApplied(TestSuite suite) =>
suite.path?.contains(_testRunnerFileName) == true;

String? _firstGroupName(Test test, Map<int, TestGroup> groups) => test.groupIDs
.map((groupID) => groups[groupID]?.name)
.firstWhereOrNull((groupName) => groupName?.isNotEmpty == true);

String _actualTestPath(
bool optimizationApplied,
String path,
String? groupName,
) =>
optimizationApplied
? path.replaceFirst(_testRunnerFileName, groupName!)
: path;

String _actualTestName(
bool optimizationApplied,
String name,
String? firstGroupName,
) =>
optimizationApplied ? name.replaceFirst(firstGroupName!, '').trim() : name;

const _testRunnerFileName = '.test_runner.dart';

final int _lineLength = () {
try {
return stdout.terminalColumns;
Expand Down Expand Up @@ -497,22 +540,3 @@ extension on String {
return '...$truncated';
}
}

String? _getTraceLocation({
required String stackTrace,
}) {
final trace = Trace.parse(stackTrace);
if (trace.frames.isEmpty) {
return null;
}

final lastFrame = trace.frames.last;

final library = lastFrame.library;
final line = lastFrame.line;
final column = lastFrame.column;

if (line == null) return library;
if (column == null) return '$library:$line';
return '$library:$line:$column';
}
2 changes: 1 addition & 1 deletion lib/src/commands/test/templates/test_runner_bundle.dart
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

137 changes: 131 additions & 6 deletions test/src/cli/flutter_cli_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,10 @@ void main() {
test('throws when there is no pubspec.yaml', () {
ProcessOverrides.runZoned(
() => expectLater(
Flutter.packagesGet(cwd: Directory.systemTemp.path, logger: logger),
Flutter.packagesGet(
cwd: Directory.systemTemp.path,
logger: logger,
),
throwsA(isA<PubspecNotFound>()),
),
runProcess: process.run,
Expand All @@ -113,7 +116,10 @@ void main() {

ProcessOverrides.runZoned(
() => expectLater(
Flutter.packagesGet(cwd: Directory.systemTemp.path, logger: logger),
Flutter.packagesGet(
cwd: Directory.systemTemp.path,
logger: logger,
),
throwsException,
),
runProcess: process.run,
Expand Down Expand Up @@ -361,7 +367,9 @@ void main() {

controller
..add(const DoneTestEvent(success: true, time: 0))
..add(const ExitTestEvent(exitCode: 0, time: 0));
..add(
const ExitTestEvent(exitCode: 0, time: 0),
);

await Future<void>.delayed(Duration.zero);

Expand Down Expand Up @@ -459,7 +467,8 @@ void main() {
'test/counter/cubit/counter_cubit_test.dart 16:7 main.<fn>.<fn>\n',
'\x1B[2K\rCounterCubit initial state is 0 /my_app/test/counter/cubit/counter_cubit_test.dart (FAILED)',
'\x1B[2K\rFailing Tests:\n'
'\x1B[2K\r - [FAILED] test/counter/cubit/counter_cubit_test.dart:16:7\n'
'\x1B[2K\r - /my_app/test/counter/cubit/counter_cubit_test.dart \n'
'\x1B[2K\r \t- [FAILED] CounterCubit initial state is 0\n',
],
),
);
Expand Down Expand Up @@ -528,7 +537,8 @@ void main() {
'The test description was: renders CounterPage',
'\x1B[2K\rApp renders CounterPage /my_app/test/app/view/app_test.dart (FAILED)',
'\x1B[2K\rFailing Tests:\n'
'''\x1B[2K\r - [ERROR] ...failed. See exception logs above. The test description was: renders CounterPage\n'''
'\x1B[2K\r - /my_app/test/app/view/app_test.dart \n'
'\x1B[2K\r \t- [ERROR] App renders CounterPage\n',
]),
);
directory.delete(recursive: true).ignore();
Expand Down Expand Up @@ -568,6 +578,36 @@ void main() {
stderr: stderrLogs.add,
testRunner: testRunner(
Stream.fromIterable([
const SuiteTestEvent(
suite: TestSuite(
id: 4,
platform: 'vm',
path: '/my_app/test/app/view/app_test.dart',
),
time: 0,
),
GroupTestEvent(
group: TestGroup(
id: 10,
suiteID: 4,
name: 'CounterCubit',
metadata: TestMetadata(
skip: false,
),
testCount: 1,
),
time: 0,
),
TestStartEvent(
test: Test(
id: 0,
name: 'CounterCubit emits [1] when increment is called',
suiteID: 4,
groupIDs: [10],
metadata: TestMetadata(skip: false),
),
time: 0,
),
ErrorTestEvent(
testID: 0,
error: 'error',
Expand All @@ -591,7 +631,8 @@ void main() {
'\x1B[2K\rerror',
'\x1B[2K\rtest/example_test.dart 4 main\n',
'\x1B[2K\rFailing Tests:\n'
'\x1B[2K\r - [FAILED] test/example_test.dart:4\n'
'\x1B[2K\r - /my_app/test/app/view/app_test.dart \n'
'''\x1B[2K\r \t- [FAILED] CounterCubit emits [1] when increment is called\n'''
]),
);
directory.delete(recursive: true).ignore();
Expand Down Expand Up @@ -928,6 +969,90 @@ void main() {
verify(() => progress.complete()).called(1);
directory.delete(recursive: true).ignore();
});

test('runs tests w/optimizations (failing)', () async {
final directory = Directory.systemTemp.createTempSync();
File(p.join(directory.path, 'pubspec.yaml')).createSync();
Directory(p.join(directory.path, 'test')).createSync();
await expectLater(
Flutter.test(
cwd: directory.path,
stdout: stdoutLogs.add,
stderr: stderrLogs.add,
testRunner: testRunner(
Stream.fromIterable([
const SuiteTestEvent(
suite: TestSuite(
id: 4,
platform: 'vm',
path: '/my_app/test/.test_runner.dart',
),
time: 0,
),
GroupTestEvent(
group: TestGroup(
id: 10,
suiteID: 4,
name: 'app/view/app_test.dart',
metadata: TestMetadata(
skip: false,
),
testCount: 1,
),
time: 0,
),
GroupTestEvent(
group: TestGroup(
id: 99,
suiteID: 4,
name: 'app/view/app_test.dart CounterCubit',
metadata: TestMetadata(
skip: false,
),
testCount: 1,
),
time: 0,
),
TestStartEvent(
test: Test(
id: 0,
name:
'app/view/app_test.dart CounterCubit emits [1] when increment is called',
suiteID: 4,
groupIDs: [10, 99],
metadata: TestMetadata(skip: false),
),
time: 0,
),
ErrorTestEvent(
testID: 0,
error: 'error',
stackTrace:
stack_trace.Trace.parse('test/example_test.dart 4 main')
.toString(),
isFailure: true,
time: 0,
),
const DoneTestEvent(success: false, time: 0),
const ExitTestEvent(exitCode: 1, time: 0),
]),
),
logger: logger,
),
completion(equals([ExitCode.unavailable.code])),
);
expect(
stderrLogs,
equals([
'\x1B[2K\rerror',
'\x1B[2K\rtest/example_test.dart 4 main\n',
'\x1B[2K\rFailing Tests:\n'
'\x1B[2K\r - /my_app/test/app/view/app_test.dart \n'
'''\x1B[2K\r \t- [FAILED] CounterCubit emits [1] when increment is called\n'''
]),
);
directory.delete(recursive: true).ignore();
});
});
});
}

0 comments on commit 3730d93

Please sign in to comment.