From 254ae590ae0b63fc8544246be13174afb566dc0d Mon Sep 17 00:00:00 2001 From: Abdelaziz GACEMI Date: Thu, 10 Nov 2022 23:44:03 +0100 Subject: [PATCH] feat(test): enhance logs and final test report in case of failing tests --- lib/src/cli/cli.dart | 3 +- lib/src/cli/flutter_cli.dart | 100 ++++++++----- .../test/templates/test_runner_bundle.dart | 2 +- pubspec.yaml | 2 + test/src/cli/flutter_cli_test.dart | 137 +++++++++++++++++- 5 files changed, 197 insertions(+), 47 deletions(-) diff --git a/lib/src/cli/cli.dart b/lib/src/cli/cli.dart index 1076bc4d5..a82e22024 100644 --- a/lib/src/cli/cli.dart +++ b/lib/src/cli/cli.dart @@ -1,12 +1,11 @@ import 'dart:async'; - +import 'package:collection/collection.dart'; import 'package:glob/glob.dart'; import 'package:lcov_parser/lcov_parser.dart'; import 'package:mason/mason.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:pubspec_parse/pubspec_parse.dart'; -import 'package:stack_trace/stack_trace.dart'; import 'package:universal_io/io.dart'; import 'package:very_good_cli/src/commands/test/templates/test_runner_bundle.dart'; import 'package:very_good_test_runner/very_good_test_runner.dart'; diff --git a/lib/src/cli/flutter_cli.dart b/lib/src/cli/flutter_cli.dart index baeab4cc8..3a0d946de 100644 --- a/lib/src/cli/flutter_cli.dart +++ b/lib/src/cli/flutter_cli.dart @@ -1,5 +1,7 @@ part of 'cli.dart'; +const _testRunnerFileName = '.test_runner.dart'; + /// Thrown when `flutter packages get` or `flutter pub get` /// is executed without a `pubspec.yaml`. class PubspecNotFound implements Exception {} @@ -205,13 +207,13 @@ class Flutter { '--test-randomize-ordering-seed', randomSeed ], - if (optimizePerformance) p.join('test', '.test_runner.dart') + if (optimizePerformance) p.join('test', _testRunnerFileName) ], stdout: stdout ?? noop, stderr: stderr ?? noop, ).whenComplete(() { if (optimizePerformance) { - File(p.join(cwd, 'test', '.test_runner.dart')).delete().ignore(); + File(p.join(cwd, 'test', _testRunnerFileName)).delete().ignore(); } }); }, @@ -313,14 +315,15 @@ Future _flutterTest({ final suites = {}; final groups = {}; final tests = {}; - final failedTestErrorMessages = {}; + final failedTestErrorMessages = >{}; var successCount = 0; var skipCount = 0; String computeStats() { final passingTests = successCount.formatSuccess(); - final failingTests = failedTestErrorMessages.length.formatFailure(); + final failingTests = + failedTestErrorMessages.values.expand((e) => e).length.formatFailure(); final skippedTests = skipCount.formatSkipped(); final result = [passingTests, failingTests, skippedTests] ..removeWhere((element) => element.isEmpty); @@ -367,16 +370,21 @@ Future _flutterTest({ if (event.stackTrace.trim().isNotEmpty) { stderr('$clearLine${event.stackTrace}'); } - - final traceLocation = _getTraceLocation(stackTrace: event.stackTrace); - - // When failing to recover the location from the stack trace, - // save a short description of the error - final testErrorDescription = traceLocation ?? - event.error.replaceAll('\n', ' ').truncated(_lineLength); - + final test = tests[event.testID]!; + final suite = suites[test.suiteID]!; final prefix = event.isFailure ? '[FAILED]' : '[ERROR]'; - failedTestErrorMessages[event.testID] = '$prefix $testErrorDescription'; + + final optimizationApplied = _isOptimizationApplied(suite); + final topGroupName = _topGroupName(test, groups); + final testPath = + _actualTestPath(optimizationApplied, suite.path!, topGroupName); + final testName = + _actualTestName(optimizationApplied, test.name, topGroupName); + + failedTestErrorMessages[testPath] = [ + ...failedTestErrorMessages[testPath] ?? [], + '$prefix $testName' + ]; } if (event is TestDoneEvent) { @@ -384,24 +392,30 @@ Future _flutterTest({ final test = tests[event.testID]!; final suite = suites[test.suiteID]!; + final optimizationApplied = _isOptimizationApplied(suite); + final firstGroupName = _topGroupName(test, groups); + final testPath = + _actualTestPath(optimizationApplied, suite.path!, firstGroupName); + final testName = + _actualTestName(optimizationApplied, test.name, firstGroupName); if (event.skipped) { stdout( - '''$clearLine${lightYellow.wrap('${test.name} ${suite.path} (SKIPPED)')}\n''', + '''$clearLine${lightYellow.wrap('$testName $testPath (SKIPPED)')}\n''', ); skipCount++; } else if (event.result == TestResult.success) { successCount++; } else { - stderr('$clearLine${test.name} ${suite.path} (FAILED)'); + stderr('$clearLine$testName $testPath (FAILED)'); } final timeElapsed = Duration(milliseconds: event.time).formatted(); final stats = computeStats(); - final testName = test.name.toSingleLine().truncated( + final truncatedTestName = testName.toSingleLine().truncated( _lineLength - (timeElapsed.length + stats.length + 2), ); - stdout('''$clearLine$timeElapsed $stats: $testName'''); + stdout('''$clearLine$timeElapsed $stats: $truncatedTestName'''); } if (event is DoneTestEvent) { @@ -422,9 +436,15 @@ Future _flutterTest({ final title = styleBold.wrap('Failing Tests:'); final lines = StringBuffer('$clearLine$title\n'); - for (final errorMessage in failedTestErrorMessages.values) { - lines.writeln('$clearLine - $errorMessage'); + for (final testSuiteErrorMessages + in failedTestErrorMessages.entries) { + lines.writeln('$clearLine - ${testSuiteErrorMessages.key} '); + + for (final errorMessage in testSuiteErrorMessages.value) { + lines.writeln('$clearLine \t- $errorMessage'); + } } + stderr(lines.toString()); } } @@ -448,6 +468,29 @@ Future _flutterTest({ return completer.future; } +bool _isOptimizationApplied(TestSuite suite) => + suite.path?.contains(_testRunnerFileName) ?? false; + +String? _topGroupName(Test test, Map groups) => test.groupIDs + .map((groupID) => groups[groupID]?.name) + .firstWhereOrNull((groupName) => groupName?.isNotEmpty ?? false); + +String _actualTestPath( + bool optimizationApplied, + String path, + String? groupName, +) => + optimizationApplied + ? path.replaceFirst(_testRunnerFileName, groupName!) + : path; + +String _actualTestName( + bool optimizationApplied, + String name, + String? topGroupName, +) => + optimizationApplied ? name.replaceFirst(topGroupName!, '').trim() : name; + final int _lineLength = () { try { return stdout.terminalColumns; @@ -500,22 +543,3 @@ extension on String { String toSingleLine() => replaceAll('\n', '').replaceAll(RegExp(r'\s\s+'), ' '); } - -String? _getTraceLocation({ - required String stackTrace, -}) { - final trace = Trace.parse(stackTrace); - if (trace.frames.isEmpty) { - return null; - } - - final lastFrame = trace.frames.last; - - final library = lastFrame.library; - final line = lastFrame.line; - final column = lastFrame.column; - - if (line == null) return library; - if (column == null) return '$library:$line'; - return '$library:$line:$column'; -} diff --git a/lib/src/commands/test/templates/test_runner_bundle.dart b/lib/src/commands/test/templates/test_runner_bundle.dart index 2241f402b..8276c62e1 100644 --- a/lib/src/commands/test/templates/test_runner_bundle.dart +++ b/lib/src/commands/test/templates/test_runner_bundle.dart @@ -8,7 +8,7 @@ final testRunnerBundle = MasonBundle.fromJson({ { "path": "test/.test_runner.dart", "data": - "Ly8gR0VORVJBVEVEIENPREUgLSBETyBOT1QgTU9ESUZZIEJZIEhBTkQKLy8gQ29uc2lkZXIgYWRkaW5nIHRoaXMgZmlsZSB0byB5b3VyIC5naXRpZ25vcmUuCgp7eyNpc0ZsdXR0ZXJ9fWltcG9ydCAnZGFydDppbyc7CgppbXBvcnQgJ3BhY2thZ2U6Zmx1dHRlcl90ZXN0L2ZsdXR0ZXJfdGVzdC5kYXJ0JzsKe3svaXNGbHV0dGVyfX17e15pc0ZsdXR0ZXJ9fWltcG9ydCAncGFja2FnZTp0ZXN0L3Rlc3QuZGFydCc7e3svaXNGbHV0dGVyfX0KCnt7I3Rlc3RzfX1pbXBvcnQgJ3t7ey59fX0nIGFzIHt7I3NuYWtlQ2FzZX19e3t7Ln19fXt7L3NuYWtlQ2FzZX19Owp7ey90ZXN0c319CnZvaWQgbWFpbigpIHsKe3sjaXNGbHV0dGVyfX0gIGdvbGRlbkZpbGVDb21wYXJhdG9yID0gX1Rlc3RPcHRpbWl6YXRpb25Bd2FyZUdvbGRlbkZpbGVDb21wYXJhdG9yKCk7e3svaXNGbHV0dGVyfX0Ke3sjdGVzdHN9fSAgZ3JvdXAoJ3t7I3NuYWtlQ2FzZX19e3t7Ln19fXt7L3NuYWtlQ2FzZX19JywgKCkgeyB7eyNzbmFrZUNhc2V9fXt7ey59fX17ey9zbmFrZUNhc2V9fS5tYWluKCk7IH0pOwp7ey90ZXN0c319fQoKe3sjaXNGbHV0dGVyfX0KY2xhc3MgX1Rlc3RPcHRpbWl6YXRpb25Bd2FyZUdvbGRlbkZpbGVDb21wYXJhdG9yIGV4dGVuZHMgTG9jYWxGaWxlQ29tcGFyYXRvciB7CiAgZmluYWwgTGlzdDxTdHJpbmc+IGdvbGRlbkZpbGVQYXRoczsKCiAgX1Rlc3RPcHRpbWl6YXRpb25Bd2FyZUdvbGRlbkZpbGVDb21wYXJhdG9yKCkKICAgICAgOiBnb2xkZW5GaWxlUGF0aHMgPSBfZ29sZGVuRmlsZVBhdGhzLAogICAgICAgIHN1cGVyKF90ZXN0RmlsZSk7CgogIHN0YXRpYyBVcmkgZ2V0IF90ZXN0RmlsZSB7CiAgICBmaW5hbCBiYXNlZGlyID0KICAgICAgICAoZ29sZGVuRmlsZUNvbXBhcmF0b3IgYXMgTG9jYWxGaWxlQ29tcGFyYXRvcikuYmFzZWRpci50b1N0cmluZygpOwogICAgcmV0dXJuIFVyaS5wYXJzZSgiJGJhc2VkaXIvLnRlc3RfcnVubmVyLmRhcnQiKTsKICB9CgogIHN0YXRpYyBMaXN0PFN0cmluZz4gZ2V0IF9nb2xkZW5GaWxlUGF0aHMgPT4KICAgICAgRGlyZWN0b3J5LmZyb21VcmkoKGdvbGRlbkZpbGVDb21wYXJhdG9yIGFzIExvY2FsRmlsZUNvbXBhcmF0b3IpLmJhc2VkaXIpCiAgICAgICAgICAubGlzdFN5bmMocmVjdXJzaXZlOiB0cnVlLCBmb2xsb3dMaW5rczogdHJ1ZSkKICAgICAgICAgIC53aGVyZVR5cGU8RmlsZT4oKQogICAgICAgICAgLm1hcCgoZmlsZSkgPT4gZmlsZS5wYXRoKQogICAgICAgICAgLndoZXJlKChwYXRoKSA9PiBwYXRoLmVuZHNXaXRoKCcucG5nJykpCiAgICAgICAgICAudG9MaXN0KCk7CgogIEBvdmVycmlkZQogIFVyaSBnZXRUZXN0VXJpKFVyaSBrZXksIGludD8gdmVyc2lvbikgewogICAgZmluYWwga2V5U3RyaW5nID0ga2V5LnBhdGg7CiAgICByZXR1cm4gVXJpLnBhcnNlKGdvbGRlbkZpbGVQYXRocwogICAgICAgIC5zaW5nbGVXaGVyZSgoZ29sZGVuRmlsZVBhdGgpID0+IGdvbGRlbkZpbGVQYXRoLmVuZHNXaXRoKGtleVN0cmluZykpKTsKICB9Cn0Ke3svaXNGbHV0dGVyfX0=", + "Ly8gR0VORVJBVEVEIENPREUgLSBETyBOT1QgTU9ESUZZIEJZIEhBTkQKLy8gQ29uc2lkZXIgYWRkaW5nIHRoaXMgZmlsZSB0byB5b3VyIC5naXRpZ25vcmUuCgp7eyNpc0ZsdXR0ZXJ9fWltcG9ydCAnZGFydDppbyc7CgppbXBvcnQgJ3BhY2thZ2U6Zmx1dHRlcl90ZXN0L2ZsdXR0ZXJfdGVzdC5kYXJ0JzsKe3svaXNGbHV0dGVyfX17e15pc0ZsdXR0ZXJ9fWltcG9ydCAncGFja2FnZTp0ZXN0L3Rlc3QuZGFydCc7e3svaXNGbHV0dGVyfX0KCnt7I3Rlc3RzfX1pbXBvcnQgJ3t7ey59fX0nIGFzIHt7I3NuYWtlQ2FzZX19e3t7Ln19fXt7L3NuYWtlQ2FzZX19Owp7ey90ZXN0c319CnZvaWQgbWFpbigpIHsKe3sjaXNGbHV0dGVyfX0gIGdvbGRlbkZpbGVDb21wYXJhdG9yID0gX1Rlc3RPcHRpbWl6YXRpb25Bd2FyZUdvbGRlbkZpbGVDb21wYXJhdG9yKCk7e3svaXNGbHV0dGVyfX0Ke3sjdGVzdHN9fSAgZ3JvdXAoJ3t7ey59fX0nLCAoKSB7IHt7I3NuYWtlQ2FzZX19e3t7Ln19fXt7L3NuYWtlQ2FzZX19Lm1haW4oKTsgfSk7Cnt7L3Rlc3RzfX19Cgp7eyNpc0ZsdXR0ZXJ9fQpjbGFzcyBfVGVzdE9wdGltaXphdGlvbkF3YXJlR29sZGVuRmlsZUNvbXBhcmF0b3IgZXh0ZW5kcyBMb2NhbEZpbGVDb21wYXJhdG9yIHsKICBmaW5hbCBMaXN0PFN0cmluZz4gZ29sZGVuRmlsZVBhdGhzOwoKICBfVGVzdE9wdGltaXphdGlvbkF3YXJlR29sZGVuRmlsZUNvbXBhcmF0b3IoKQogICAgICA6IGdvbGRlbkZpbGVQYXRocyA9IF9nb2xkZW5GaWxlUGF0aHMsCiAgICAgICAgc3VwZXIoX3Rlc3RGaWxlKTsKCiAgc3RhdGljIFVyaSBnZXQgX3Rlc3RGaWxlIHsKICAgIGZpbmFsIGJhc2VkaXIgPQogICAgICAgIChnb2xkZW5GaWxlQ29tcGFyYXRvciBhcyBMb2NhbEZpbGVDb21wYXJhdG9yKS5iYXNlZGlyLnRvU3RyaW5nKCk7CiAgICByZXR1cm4gVXJpLnBhcnNlKCIkYmFzZWRpci8udGVzdF9ydW5uZXIuZGFydCIpOwogIH0KCiAgc3RhdGljIExpc3Q8U3RyaW5nPiBnZXQgX2dvbGRlbkZpbGVQYXRocyA9PgogICAgICBEaXJlY3RvcnkuZnJvbVVyaSgoZ29sZGVuRmlsZUNvbXBhcmF0b3IgYXMgTG9jYWxGaWxlQ29tcGFyYXRvcikuYmFzZWRpcikKICAgICAgICAgIC5saXN0U3luYyhyZWN1cnNpdmU6IHRydWUsIGZvbGxvd0xpbmtzOiB0cnVlKQogICAgICAgICAgLndoZXJlVHlwZTxGaWxlPigpCiAgICAgICAgICAubWFwKChmaWxlKSA9PiBmaWxlLnBhdGgpCiAgICAgICAgICAud2hlcmUoKHBhdGgpID0+IHBhdGguZW5kc1dpdGgoJy5wbmcnKSkKICAgICAgICAgIC50b0xpc3QoKTsKCiAgQG92ZXJyaWRlCiAgVXJpIGdldFRlc3RVcmkoVXJpIGtleSwgaW50PyB2ZXJzaW9uKSB7CiAgICBmaW5hbCBrZXlTdHJpbmcgPSBrZXkucGF0aDsKICAgIHJldHVybiBVcmkucGFyc2UoZ29sZGVuRmlsZVBhdGhzCiAgICAgICAgLnNpbmdsZVdoZXJlKChnb2xkZW5GaWxlUGF0aCkgPT4gZ29sZGVuRmlsZVBhdGguZW5kc1dpdGgoa2V5U3RyaW5nKSkpOwogIH0KfQp7ey9pc0ZsdXR0ZXJ9fQ==", "type": "text" } ], diff --git a/pubspec.yaml b/pubspec.yaml index 1704e5c0c..28f3d5257 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,6 +9,7 @@ environment: dependencies: args: ^2.1.0 cli_completion: ^0.2.0 + collection: ^1.17.1 glob: ^2.0.2 lcov_parser: ^0.1.2 mason: ">=0.1.0-dev.41 <0.1.0-dev.42" @@ -22,6 +23,7 @@ dependencies: usage: ^4.0.2 very_good_test_runner: ^0.1.2 + dev_dependencies: build_runner: ^2.0.0 build_verify: ^3.0.0 diff --git a/test/src/cli/flutter_cli_test.dart b/test/src/cli/flutter_cli_test.dart index 158335f1b..036c31730 100644 --- a/test/src/cli/flutter_cli_test.dart +++ b/test/src/cli/flutter_cli_test.dart @@ -90,7 +90,10 @@ void main() { test('throws when there is no pubspec.yaml', () { ProcessOverrides.runZoned( () => expectLater( - Flutter.packagesGet(cwd: Directory.systemTemp.path, logger: logger), + Flutter.packagesGet( + cwd: Directory.systemTemp.path, + logger: logger, + ), throwsA(isA()), ), runProcess: process.run, @@ -113,7 +116,10 @@ void main() { ProcessOverrides.runZoned( () => expectLater( - Flutter.packagesGet(cwd: Directory.systemTemp.path, logger: logger), + Flutter.packagesGet( + cwd: Directory.systemTemp.path, + logger: logger, + ), throwsException, ), runProcess: process.run, @@ -361,7 +367,9 @@ void main() { controller ..add(const DoneTestEvent(success: true, time: 0)) - ..add(const ExitTestEvent(exitCode: 0, time: 0)); + ..add( + const ExitTestEvent(exitCode: 0, time: 0), + ); await Future.delayed(Duration.zero); @@ -459,7 +467,8 @@ void main() { 'test/counter/cubit/counter_cubit_test.dart 16:7 main..\n', '\x1B[2K\rCounterCubit initial state is 0 /my_app/test/counter/cubit/counter_cubit_test.dart (FAILED)', '\x1B[2K\rFailing Tests:\n' - '\x1B[2K\r - [FAILED] test/counter/cubit/counter_cubit_test.dart:16:7\n' + '\x1B[2K\r - /my_app/test/counter/cubit/counter_cubit_test.dart \n' + '\x1B[2K\r \t- [FAILED] CounterCubit initial state is 0\n', ], ), ); @@ -529,7 +538,8 @@ void main() { 'The test description was: renders CounterPage', '\x1B[2K\rApp renders CounterPage /my_app/test/app/view/app_test.dart (FAILED)', '\x1B[2K\rFailing Tests:\n' - '''\x1B[2K\r - [ERROR] ...failed. See exception logs above. The test description was: renders CounterPage\n''' + '\x1B[2K\r - /my_app/test/app/view/app_test.dart \n' + '\x1B[2K\r \t- [ERROR] App renders CounterPage\n', ]), ); directory.delete(recursive: true).ignore(); @@ -569,6 +579,36 @@ void main() { stderr: stderrLogs.add, testRunner: testRunner( Stream.fromIterable([ + const SuiteTestEvent( + suite: TestSuite( + id: 4, + platform: 'vm', + path: '/my_app/test/app/view/app_test.dart', + ), + time: 0, + ), + GroupTestEvent( + group: TestGroup( + id: 10, + suiteID: 4, + name: 'CounterCubit', + metadata: TestMetadata( + skip: false, + ), + testCount: 1, + ), + time: 0, + ), + TestStartEvent( + test: Test( + id: 0, + name: 'CounterCubit emits [1] when increment is called', + suiteID: 4, + groupIDs: [10], + metadata: TestMetadata(skip: false), + ), + time: 0, + ), ErrorTestEvent( testID: 0, error: 'error', @@ -592,7 +632,8 @@ void main() { '\x1B[2K\rerror', '\x1B[2K\rtest/example_test.dart 4 main\n', '\x1B[2K\rFailing Tests:\n' - '\x1B[2K\r - [FAILED] test/example_test.dart:4\n' + '\x1B[2K\r - /my_app/test/app/view/app_test.dart \n' + '''\x1B[2K\r \t- [FAILED] CounterCubit emits [1] when increment is called\n''' ]), ); directory.delete(recursive: true).ignore(); @@ -929,6 +970,90 @@ void main() { verify(() => progress.complete()).called(1); directory.delete(recursive: true).ignore(); }); + + test('runs tests w/optimizations (failing)', () async { + final directory = Directory.systemTemp.createTempSync(); + File(p.join(directory.path, 'pubspec.yaml')).createSync(); + Directory(p.join(directory.path, 'test')).createSync(); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: stdoutLogs.add, + stderr: stderrLogs.add, + testRunner: testRunner( + Stream.fromIterable([ + const SuiteTestEvent( + suite: TestSuite( + id: 4, + platform: 'vm', + path: '/my_app/test/.test_runner.dart', + ), + time: 0, + ), + GroupTestEvent( + group: TestGroup( + id: 10, + suiteID: 4, + name: 'app/view/app_test.dart', + metadata: TestMetadata( + skip: false, + ), + testCount: 1, + ), + time: 0, + ), + GroupTestEvent( + group: TestGroup( + id: 99, + suiteID: 4, + name: 'app/view/app_test.dart CounterCubit', + metadata: TestMetadata( + skip: false, + ), + testCount: 1, + ), + time: 0, + ), + TestStartEvent( + test: Test( + id: 0, + name: + 'app/view/app_test.dart CounterCubit emits [1] when increment is called', + suiteID: 4, + groupIDs: [10, 99], + metadata: TestMetadata(skip: false), + ), + time: 0, + ), + ErrorTestEvent( + testID: 0, + error: 'error', + stackTrace: + stack_trace.Trace.parse('test/example_test.dart 4 main') + .toString(), + isFailure: true, + time: 0, + ), + const DoneTestEvent(success: false, time: 0), + const ExitTestEvent(exitCode: 1, time: 0), + ]), + ), + logger: logger, + ), + completion(equals([ExitCode.unavailable.code])), + ); + expect( + stderrLogs, + equals([ + '\x1B[2K\rerror', + '\x1B[2K\rtest/example_test.dart 4 main\n', + '\x1B[2K\rFailing Tests:\n' + '\x1B[2K\r - /my_app/test/app/view/app_test.dart \n' + '''\x1B[2K\r \t- [FAILED] CounterCubit emits [1] when increment is called\n''' + ]), + ); + directory.delete(recursive: true).ignore(); + }); }); }); }