Support passing extra arguments to test via test_with_coverage

Fixes #403

Also cleaned up help text

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.5.0
+
+* Support passing extra arguments to `test_with_coverage` which are then passed
+  to `package:test`.
+
+  Example: `dart run coverage:test_with_coverage -- --preset CI`
+
+
 ## 1.4.0 - 2022-6-16
 
 * Added `HitMap.parseJsonSync` which takes a cache of ignored lines which can

bin/test_with_coverage.dart

@@ -90,8 +90,9 @@ class Flags {
     this.port,
     this.testScript,
     this.functionCoverage,
-    this.branchCoverage,
-  );
+    this.branchCoverage, {
+    required this.rest,
+  });
 
   final String packageDir;
   final String packageName;
@@ -100,18 +101,24 @@ class Flags {
   final String testScript;
   final bool functionCoverage;
   final bool branchCoverage;
+  final List<String> rest;
 }
 
 Future<Flags> _parseArgs(List<String> arguments) async {
   final parser = _createArgParser();
   final args = parser.parse(arguments);
 
   void printUsage() {
-    print('Runs tests and collects coverage for a package. By default this '
-        "script assumes it's being run from the root directory of a package, and "
-        'outputs a coverage.json and lcov.info to ./coverage/');
-    print('Usage: dart test_with_coverage.dart [OPTIONS...]\n');
-    print(parser.usage);
+    print('''
+Runs tests and collects coverage for a package.
+
+By default this  script assumes it's being run from the root directory of a
+package, and outputs a coverage.json and lcov.info to ./coverage/
+
+Usage: test_with_coverage [OPTIONS...] [-- <test script OPTIONS>]
+
+${parser.usage}
+''');
   }
 
   Never fail(String msg) {
@@ -147,6 +154,7 @@ Future<Flags> _parseArgs(List<String> arguments) async {
     args['test'] as String,
     args['function-coverage'] as bool,
     args['branch-coverage'] as bool,
+    rest: args.rest,
   );
 }
 
@@ -166,21 +174,25 @@ Future<void> main(List<String> arguments) async {
   }
 
   final serviceUriCompleter = Completer<Uri>();
-  final testProcess = _dartRun([
-    if (flags.branchCoverage) '--branch-coverage',
-    'run',
-    '--pause-isolates-on-exit',
-    '--disable-service-auth-codes',
-    '--enable-vm-service=${flags.port}',
-    flags.testScript,
-  ], onStdout: (line) {
-    if (!serviceUriCompleter.isCompleted) {
-      final uri = extractVMServiceUri(line);
-      if (uri != null) {
-        serviceUriCompleter.complete(uri);
+  final testProcess = _dartRun(
+    [
+      if (flags.branchCoverage) '--branch-coverage',
+      'run',
+      '--pause-isolates-on-exit',
+      '--disable-service-auth-codes',
+      '--enable-vm-service=${flags.port}',
+      flags.testScript,
+      ...flags.rest,
+    ],
+    onStdout: (line) {
+      if (!serviceUriCompleter.isCompleted) {
+        final uri = extractVMServiceUri(line);
+        if (uri != null) {
+          serviceUriCompleter.complete(uri);
+        }
       }
-    }
-  });
+    },
+  );
   final serviceUri = await serviceUriCompleter.future;
 
   await collect_coverage.main([

pubspec.yaml

@@ -1,5 +1,5 @@
 name: coverage
-version: 1.4.0
+version: 1.5.0
 description: Coverage data manipulation and formatting
 repository: https://github.com/dart-lang/coverage
 

test/test_with_coverage_package/lib/validate_lib.dart

@@ -0,0 +1,15 @@
+int sum(Iterable<int> values) {
+  var val = 0;
+  for (var value in values) {
+    val += value;
+  }
+  return val;
+}
+
+int product(Iterable<int> values) {
+  var val = 1;
+  for (var value in values) {
+    val *= value;
+  }
+  return val;
+}

test/test_with_coverage_package/pubspec.yaml

@@ -4,6 +4,9 @@ publish_to: none
 environment:
   sdk: '>=2.15.0 <3.0.0'
 
-dependencies:
+dev_dependencies:
+  test: ^1.16.0
+
+dependency_overrides:
   coverage:
     path: ../../

test/test_with_coverage_package/test/product_test.dart

@@ -0,0 +1,14 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+// ignore: avoid_relative_lib_imports
+import '../lib/validate_lib.dart';
+
+void main() {
+  test('product', () {
+    expect(product([2, 3]), 6);
+  });
+}

test/test_with_coverage_package/main.dart → test/test_with_coverage_package/test/sum_test.dart

@@ -2,4 +2,13 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-void main() {}
+import 'package:test/test.dart';
+
+// ignore: avoid_relative_lib_imports
+import '../lib/validate_lib.dart';
+
+void main() {
+  test('sum', () {
+    expect(sum([1, 2]), 3);
+  });
+}

test/test_with_coverage_test.dart

@@ -2,19 +2,22 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:path/path.dart' as p;
 import 'package:test/test.dart';
+import 'package:test_descriptor/test_descriptor.dart' as d;
 import 'package:test_process/test_process.dart';
 
+import 'test_util.dart';
+
 // this package
 final _pkgDir = p.absolute('');
 final _testWithCoveragePath = p.join(_pkgDir, 'bin', 'test_with_coverage.dart');
 
 // test package
 final _testPkgDirPath = p.join(_pkgDir, 'test', 'test_with_coverage_package');
-final _testPkgExePath = p.join(_testPkgDirPath, 'main.dart');
 
 /// Override PUB_CACHE
 ///
@@ -24,59 +27,128 @@ final _testPkgExePath = p.join(_testPkgDirPath, 'main.dart');
 final _pubCachePathInTestPkgSubDir = p.join(_pkgDir, 'var', 'pub-cache');
 final _env = {'PUB_CACHE': _pubCachePathInTestPkgSubDir};
 
+const _testPackageName = 'coverage_integration_test_for_test_with_coverage';
+
 int _port = 9300;
 
+Iterable<File> _dartFiles(String dir) =>
+    Directory(p.join(_testPkgDirPath, dir)).listSync().whereType<File>();
+
+String _fixTestFile(String content) => content.replaceAll(
+      "import '../lib/",
+      "import 'package:$_testPackageName/",
+    );
+
 void main() {
   setUpAll(() async {
+    for (var dir in const ['lib', 'test']) {
+      await d.dir(dir, [
+        for (var dartFile in _dartFiles(dir))
+          d.file(
+            p.basename(dartFile.path),
+            _fixTestFile(dartFile.readAsStringSync()),
+          ),
+      ]).create();
+    }
+
+    var pubspecContent =
+        File(p.join(_testPkgDirPath, 'pubspec.yaml')).readAsStringSync();
+
+    expect(
+      pubspecContent.replaceAll('\r\n', '\n'),
+      contains(r'''
+dependency_overrides:
+  coverage:
+    path: ../../
+'''),
+    );
+
+    pubspecContent =
+        pubspecContent.replaceFirst('path: ../../', 'path: $_pkgDir');
+
+    await d.file('pubspec.yaml', pubspecContent).create();
+
     final localPub = await _run(['pub', 'get']);
     await localPub.shouldExit(0);
-
-    final globalPub =
-        await _run(['pub', 'global', 'activate', '-s', 'path', _pkgDir]);
-    await globalPub.shouldExit(0);
   });
 
-  tearDownAll(() {
-    for (final entry in [
-      Directory(p.join(_testPkgDirPath, '.dart_tool')),
-      Directory(p.join(_testPkgDirPath, 'coverage')),
-      File(p.join(_testPkgDirPath, '.packages')),
-      File(p.join(_testPkgDirPath, 'pubspec.lock')),
-    ]) {
-      if (entry.existsSync()) {
-        entry.deleteSync(recursive: true);
-      }
-    }
+  test('dart run bin/test_with_coverage.dart -f', () async {
+    final list = await _runTest(['run', _testWithCoveragePath, '-f']);
+
+    final sources = list.sources();
+    final functionHits = functionInfoFromSources(sources);
+
+    expect(
+      functionHits['package:$_testPackageName/validate_lib.dart'],
+      {
+        'product': 1,
+        'sum': 1,
+      },
+    );
   });
 
-  test('dart run bin/test_with_coverage.dart', () async {
-    final result = await _runTest(['run', _testWithCoveragePath]);
-    await result.shouldExit(0);
+  test('dart run bin/test_with_coverage.dart -f -- -N sum', () async {
+    final list = await _runTest(
+      ['run', _testWithCoveragePath, '-f'],
+      extraArgs: ['--', '-N', 'sum'],
+    );
+
+    final sources = list.sources();
+    final functionHits = functionInfoFromSources(sources);
+
+    expect(
+      functionHits['package:$_testPackageName/validate_lib.dart'],
+      {
+        'product': 0,
+        'sum': 1,
+      },
+      reason: 'only `sum` tests should be run',
+    );
   });
 
   test('dart run coverage:test_with_coverage', () async {
-    final result = await _runTest(['run', 'coverage:test_with_coverage']);
-    await result.shouldExit(0);
+    await _runTest(['run', 'coverage:test_with_coverage']);
   });
 
   test('dart pub global run coverage:test_with_coverage', () async {
-    final result =
-        await _runTest(['pub', 'global', 'run', 'coverage:test_with_coverage']);
-    await result.shouldExit(0);
+    final globalPub =
+        await _run(['pub', 'global', 'activate', '-s', 'path', _pkgDir]);
+    await globalPub.shouldExit(0);
+
+    await _runTest(
+      ['pub', 'global', 'run', 'coverage:test_with_coverage'],
+    );
   });
 }
 
 Future<TestProcess> _run(List<String> args) => TestProcess.start(
       Platform.executable,
       args,
-      workingDirectory: _testPkgDirPath,
+      workingDirectory: d.sandbox,
       environment: _env,
     );
 
-Future<TestProcess> _runTest(List<String> invokeArgs) => _run([
-      ...invokeArgs,
-      '--port',
-      '${_port++}',
-      '--test',
-      _testPkgExePath,
-    ]);
+Future<List<Map<String, dynamic>>> _runTest(
+  List<String> invokeArgs, {
+  List<String>? extraArgs,
+}) async {
+  final process = await _run([
+    ...invokeArgs,
+    '--port',
+    '${_port++}',
+    ...?extraArgs,
+  ]);
+
+  await process.shouldExit(0);
+
+  await d.dir(
+    'coverage',
+    [d.file('coverage.json', isNotEmpty), d.file('lcov.info', isNotEmpty)],
+  ).validate();
+
+  final coverageDataFile = File(p.join(d.sandbox, 'coverage', 'coverage.json'));
+
+  final json = jsonDecode(coverageDataFile.readAsStringSync());
+
+  return coverageDataFromJson(json as Map<String, dynamic>);
+}