Skip to content

Commit

Permalink
fix(test): exit (non-zero) on test failure (#344)
Browse files Browse the repository at this point in the history
  • Loading branch information
@felangel
felangel committed Mar 29, 2022
1 parent 3c933cc commit 828fb51
Show file tree
Hide file tree
Showing 5 changed files with 69 additions and 35 deletions.
2 changes: 1 addition & 1 deletion lib/src/cli/cli.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class _Cmd {
return result;
}

static Iterable<Future> runWhere<T>({
static Iterable<Future<T>> runWhere<T>({
required Future<T> Function(FileSystemEntity) run,
required bool Function(FileSystemEntity) where,
String cwd = '.',
Expand Down
27 changes: 17 additions & 10 deletions lib/src/cli/flutter_cli.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ class Flutter {
}

/// Run tests (`flutter test`).
static Future<void> test({
/// Returns a list of exit codes for each test process.
static Future<List<int>> test({
String cwd = '.',
bool recursive = false,
bool collectCoverage = false,
Expand All @@ -131,7 +132,7 @@ class Flutter {
await lcovFile.delete();
}

await _runCommand(
final results = await _runCommand<int>(
cmd: (cwd) async {
void noop(String? _) {}
final target = DirectoryGeneratorTarget(Directory(p.normalize(cwd)));
Expand Down Expand Up @@ -199,11 +200,12 @@ class Flutter {
final coverage = coverageMetrics.percentage;
if (coverage < minCoverage) throw MinCoverageNotMet(coverage);
}
return results;
}
}

/// Run a command on directories with a `pubspec.yaml`.
Future<void> _runCommand<T>({
Future<List<T>> _runCommand<T>({
required Future<T> Function(String cwd) cmd,
required String cwd,
required bool recursive,
Expand All @@ -212,24 +214,25 @@ Future<void> _runCommand<T>({
final pubspec = File(p.join(cwd, 'pubspec.yaml'));
if (!pubspec.existsSync()) throw PubspecNotFound();

await cmd(cwd);
return;
return [await cmd(cwd)];
}

final processes = _Cmd.runWhere(
final processes = _Cmd.runWhere<T>(
run: (entity) => cmd(entity.parent.path),
where: _isPubspec,
cwd: cwd,
);

if (processes.isEmpty) throw PubspecNotFound();

final results = <T>[];
for (final process in processes) {
await process;
results.add(await process);
}
return results;
}

Future<void> _flutterTest({
Future<int> _flutterTest({
String cwd = '.',
bool collectCoverage = false,
List<String>? arguments,
Expand All @@ -238,7 +241,7 @@ Future<void> _flutterTest({
}) {
const clearLine = '\u001B[2K\r';

final completer = Completer<void>();
final completer = Completer<int>();
final suites = <int, TestSuite>{};
final groups = <int, TestGroup>{};
final tests = <int, Test>{};
Expand Down Expand Up @@ -330,7 +333,11 @@ Future<void> _flutterTest({
: lightRed.wrap('Some tests failed.')!;

stdout('$clearLine${darkGray.wrap(timeElapsed)} $stats: $summary\n');
completer.complete();
completer.complete(
event.success == true
? ExitCode.success.code
: ExitCode.unavailable.code,
);
}
},
onError: completer.completeError,
Expand Down
7 changes: 5 additions & 2 deletions lib/src/commands/test/test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import 'package:very_good_cli/src/cli/cli.dart';
typedef FlutterInstalledCommand = Future<bool> Function();

/// Signature for the [Flutter.test] method.
typedef FlutterTestCommand = Future<void> Function({
typedef FlutterTestCommand = Future<List<int>> Function({
String cwd,
bool recursive,
bool collectCoverage,
Expand Down Expand Up @@ -130,7 +130,7 @@ This command should be run from the root of your Flutter project.''',

if (isFlutterInstalled) {
try {
await _flutterTest(
final results = await _flutterTest(
optimizePerformance:
optimizePerformance && _argResults.rest.isEmpty && !updateGoldens,
recursive: recursive,
Expand All @@ -148,6 +148,9 @@ This command should be run from the root of your Flutter project.''',
..._argResults.rest,
],
);
if (results.any((code) => code != ExitCode.success.code)) {
return ExitCode.unavailable.code;
}
} on MinCoverageNotMet catch (e) {
_logger.err(
'''Expected coverage >= ${minCoverage!.toStringAsFixed(2)}% but actual is ${e.coverage.toStringAsFixed(2)}%.''',
Expand Down
43 changes: 23 additions & 20 deletions test/src/cli/flutter_cli_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.unavailable.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -366,7 +366,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -405,7 +405,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -446,7 +446,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -482,7 +482,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.unavailable.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -519,7 +519,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -552,7 +552,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.unavailable.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -581,7 +581,10 @@ void main() {
File(
p.join(testDirectory.path, 'example_test.dart'),
).writeAsStringSync(testContents);
expectLater(Flutter.test(cwd: directory.path), completes);
expectLater(
Flutter.test(cwd: directory.path),
completion(equals([ExitCode.success.code])),
);
});

test('completes when there is a test directory (passing)', () async {
Expand All @@ -598,7 +601,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -632,7 +635,7 @@ void main() {
stderr: logger.err,
progress: logger.progress,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(() => logger.progress('Optimizing tests')).called(1);
verify(
Expand Down Expand Up @@ -669,7 +672,7 @@ void main() {
stderr: logger.err,
progress: logger.progress,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(() => logger.progress('Optimizing tests')).called(1);
verify(
Expand Down Expand Up @@ -705,7 +708,7 @@ void main() {
stdout: logger.write,
stderr: logger.err,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -744,7 +747,7 @@ void main() {
stderr: logger.err,
arguments: [otherTest.path],
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -781,7 +784,7 @@ void main() {
stderr: logger.err,
randomSeed: randomSeed,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -826,7 +829,7 @@ void main() {
stderr: logger.err,
collectCoverage: true,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -870,7 +873,7 @@ void main() {
stderr: logger.err,
collectCoverage: true,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -913,7 +916,7 @@ void main() {
collectCoverage: true,
minCoverage: 100,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -1001,7 +1004,7 @@ void main() {
collectCoverage: true,
minCoverage: 100,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -1042,7 +1045,7 @@ void main() {
collectCoverage: true,
minCoverage: 50,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down Expand Up @@ -1083,7 +1086,7 @@ void main() {
collectCoverage: true,
minCoverage: 49,
),
completes,
completion(equals([ExitCode.success.code])),
);
verify(
() => logger.write(
Expand Down
25 changes: 23 additions & 2 deletions test/src/commands/test/test_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ const expectedTestUsage = [

// ignore: one_member_abstracts
abstract class FlutterTestCommand {
Future<void> call({
Future<List<int>> call({
String cwd = '.',
bool recursive = false,
bool collectCoverage = false,
Expand Down Expand Up @@ -87,7 +87,7 @@ void main() {
stdout: any(named: 'stdout'),
stderr: any(named: 'stderr'),
),
).thenAnswer((_) async {});
).thenAnswer((_) async => [0]);
when<dynamic>(() => argResults['recursive']).thenReturn(false);
when<dynamic>(() => argResults['coverage']).thenReturn(false);
when<dynamic>(() => argResults['update-goldens']).thenReturn(false);
Expand Down Expand Up @@ -152,6 +152,27 @@ void main() {
).called(1);
});

test('exits with 70 when tests do not pass', () async {
when(
() => flutterTest(
cwd: any(named: 'cwd'),
recursive: any(named: 'recursive'),
collectCoverage: any(named: 'collectCoverage'),
optimizePerformance: any(named: 'optimizePerformance'),
minCoverage: any(named: 'minCoverage'),
excludeFromCoverage: any(named: 'excludeFromCoverage'),
arguments: any(named: 'arguments'),
progress: any(named: 'progress'),
stdout: any(named: 'stdout'),
stderr: any(named: 'stderr'),
),
).thenAnswer(
(_) async => [ExitCode.success.code, ExitCode.unavailable.code],
);
final result = await testCommand.run();
expect(result, equals(ExitCode.unavailable.code));
});

test('completes normally --recursive', () async {
when<dynamic>(() => argResults['recursive']).thenReturn(true);
final result = await testCommand.run();
Expand Down

0 comments on commit 828fb51

Please sign in to comment.