Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: detect licenses locally #883

Merged
merged 25 commits into from
Nov 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
082ac2b
feat: implemented always local license check
alestiago Nov 10, 2023
62f481f
refactor: remove pubLicense injection
alestiago Nov 10, 2023
359d55d
refactor: remove pub_license.dart
alestiago Nov 10, 2023
8a61aee
feat[wip]: updating licenses_test.dart
alestiago Nov 10, 2023
0e8a2f5
test[wip]: start passing tests
alestiago Nov 10, 2023
606c311
test: use package directories
alestiago Nov 10, 2023
677ddd1
test[wip]: more passing tests
alestiago Nov 10, 2023
64b3040
test: simplify setUp
alestiago Nov 10, 2023
e32d8ef
refactor: remove old test
alestiago Nov 10, 2023
cd4a594
test[WIP]: passing even more tests
alestiago Nov 10, 2023
83a10a9
test[WIP]: even more passing tests
alestiago Nov 10, 2023
224fe58
test[WIP]: even more and more passing tests
alestiago Nov 10, 2023
30b8a4b
test: made all licenses tests pass
alestiago Nov 10, 2023
6bf7f9f
Merge branch 'main' into alestiago/always-local-license-check
alestiago Nov 10, 2023
c0cb9fe
refactor: remove testing main
alestiago Nov 10, 2023
c4abf2d
feat: failures report unknown
alestiago Nov 15, 2023
024d9d0
test: reach full coverage
alestiago Nov 15, 2023
97abfc2
feat: defined _defaultDetectionThreshold as 0.95
alestiago Nov 15, 2023
7cd5133
docs: documented ignore: implementation_imports
alestiago Nov 15, 2023
fff6b60
Merge branch 'main' into alestiago/always-local-license-check
alestiago Nov 17, 2023
077dc6d
feat: add suggestion on error failure
alestiago Nov 17, 2023
0605e03
chore: pinned pana
alestiago Nov 17, 2023
0a1bb06
Merge branch 'main' into alestiago/always-local-license-check
alestiago Nov 20, 2023
a6c999e
Update lib/src/commands/packages/commands/check/commands/licenses.dart
alestiago Nov 20, 2023
240931d
Merge branch 'main' into alestiago/always-local-license-check
alestiago Nov 20, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions lib/src/command_runner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import 'package:pub_updater/pub_updater.dart';
import 'package:universal_io/io.dart';
import 'package:very_good_cli/src/commands/commands.dart';
import 'package:very_good_cli/src/logger_extension.dart';
import 'package:very_good_cli/src/pub_license/pub_license.dart';
import 'package:very_good_cli/src/version.dart';

/// The package name.
Expand All @@ -23,7 +22,6 @@ class VeryGoodCommandRunner extends CompletionCommandRunner<int> {
Logger? logger,
PubUpdater? pubUpdater,
Map<String, String>? environment,
@visibleForTesting PubLicense? pubLicense,
}) : _logger = logger ?? Logger(),
_pubUpdater = pubUpdater ?? PubUpdater(),
_environment = environment ?? Platform.environment,
Expand All @@ -39,7 +37,7 @@ class VeryGoodCommandRunner extends CompletionCommandRunner<int> {
help: 'Noisy logging, including all shell commands executed.',
);
addCommand(CreateCommand(logger: _logger));
addCommand(PackagesCommand(logger: _logger, pubLicense: pubLicense));
addCommand(PackagesCommand(logger: _logger));
addCommand(TestCommand(logger: _logger));
addCommand(UpdateCommand(logger: _logger, pubUpdater: pubUpdater));
}
Expand Down
4 changes: 1 addition & 3 deletions lib/src/commands/packages/commands/check/check.dart
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import 'package:args/command_runner.dart';
import 'package:mason/mason.dart';
import 'package:very_good_cli/src/commands/packages/commands/check/commands/commands.dart';
import 'package:very_good_cli/src/pub_license/pub_license.dart';

/// {@template packages_check_command}
/// `very_good packages check` command for performing checks in a Dart or
Expand All @@ -11,10 +10,9 @@ class PackagesCheckCommand extends Command<int> {
/// {@macro packages_check_command}
PackagesCheckCommand({
Logger? logger,
PubLicense? pubLicense,
}) {
addSubcommand(
PackagesCheckLicensesCommand(logger: logger, pubLicense: pubLicense),
PackagesCheckLicensesCommand(logger: logger),
);
}

Expand Down
131 changes: 116 additions & 15 deletions lib/src/commands/packages/commands/check/commands/licenses.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,36 @@ import 'dart:io';

import 'package:args/args.dart';
import 'package:args/command_runner.dart';
import 'package:collection/collection.dart';
import 'package:mason/mason.dart';
import 'package:meta/meta.dart';
import 'package:package_config/package_config.dart' as package_config;

// We rely on PANA's license detection algorithm to retrieve licenses from
// packages.
//
// This license detection algorithm is not exposed as a public API, so we have
// to import it directly.
//
// See also:
//
// * [PANA's faster license detection GitHub issue](https://github.com/dart-lang/pana/issues/1277)
// ignore: implementation_imports
import 'package:pana/src/license_detection/license_detector.dart' as detector;
import 'package:path/path.dart' as path;
import 'package:pubspec_lock/pubspec_lock.dart';
import 'package:very_good_cli/src/pub_license/pub_license.dart';
import 'package:very_good_cli/src/pub_license/spdx_license.gen.dart';

/// Overrides the [package_config.findPackageConfig] function for testing.
@visibleForTesting
Future<package_config.PackageConfig?> Function(
Directory directory,
)? findPackageConfigOverride;

/// Overrides the [detector.detectLicense] function for testing.
@visibleForTesting
Future<detector.Result> Function(String, double)? detectLicenseOverride;

/// The basename of the pubspec lock file.
@visibleForTesting
const pubspecLockBasename = 'pubspec.lock';
Expand All @@ -24,6 +47,16 @@ final licenseDocumentationUri = Uri.parse(
'https://cli.vgv.dev/docs/commands/check_licenses',
);

/// The detection threshold used by [detector.detectLicense].
///
/// This value is used to determine the confidence threshold for detecting
/// licenses. The value should match the default value used by PANA.
///
/// See also:
///
/// * [PANA's default threshold value](https://github.com/dart-lang/pana/blob/b598d45051ba4e028e9021c2aeb9c04e4335de76/lib/src/license.dart#L48)
const _defaultDetectionThreshold = 0.95;

/// Defines a [Map] with dependencies as keys and their licenses as values.
///
/// If a dependency's license failed to be retrieved its license will be `null`.
Expand All @@ -40,9 +73,7 @@ class PackagesCheckLicensesCommand extends Command<int> {
/// {@macro packages_check_licenses_command}
PackagesCheckLicensesCommand({
Logger? logger,
PubLicense? pubLicense,
}) : _logger = logger ?? Logger(),
_pubLicense = pubLicense ?? PubLicense() {
}) : _logger = logger ?? Logger() {
argParser
..addFlag(
'ignore-retrieval-failures',
Expand Down Expand Up @@ -80,8 +111,6 @@ class PackagesCheckLicensesCommand extends Command<int> {

final Logger _logger;

final PubLicense _pubLicense;

@override
String get description =>
"Check packages' licenses in a Dart or Flutter project.";
Expand Down Expand Up @@ -128,6 +157,13 @@ class PackagesCheckLicensesCommand extends Command<int> {

final target = _argResults.rest.length == 1 ? _argResults.rest[0] : '.';
final targetPath = path.normalize(Directory(target).absolute.path);
final targetDirectory = Directory(targetPath);
if (!targetDirectory.existsSync()) {
_logger.err(
'''Could not find directory at $targetPath. Specify a valid path to a Dart or Flutter project.''',
);
return ExitCode.noInput.code;
}

final progress = _logger.progress('Checking licenses on $targetPath');

Expand Down Expand Up @@ -169,38 +205,86 @@ class PackagesCheckLicensesCommand extends Command<int> {
return ExitCode.usage.code;
}

final packageConfig = await _tryFindPackageConfig(targetDirectory);
if (packageConfig == null) {
progress.cancel();
_logger.err(
'''Could not find a valid package config in $targetPath. Run `dart pub get` or `flutter pub get` to generate one.''',
);
return ExitCode.noInput.code;
}

final licenses = <String, Set<String>?>{};
final detectLicense = detectLicenseOverride ?? detector.detectLicense;
for (final dependency in filteredDependencies) {
progress.update(
'''Collecting licenses from ${licenses.length + 1} out of ${filteredDependencies.length} ${filteredDependencies.length == 1 ? 'package' : 'packages'}''',
);

final dependencyName = dependency.package();
Set<String>? rawLicense;
try {
rawLicense = await _pubLicense.getLicense(dependencyName);
} on PubLicenseException catch (e) {
final errorMessage = '[$dependencyName] ${e.message}';
final cachePackageEntry = packageConfig.packages
.firstWhereOrNull((package) => package.name == dependencyName);
if (cachePackageEntry == null) {
final errorMessage =
'''[$dependencyName] Could not find cached package path. Consider running `dart pub get` or `flutter pub get` to generate a new `package_config.json`.''';
if (!ignoreFailures) {
progress.cancel();
_logger.err(errorMessage);
return ExitCode.unavailable.code;
return ExitCode.noInput.code;
}

_logger.err('\n$errorMessage');
licenses[dependencyName] = {SpdxLicense.$unknown.value};
continue;
}

final packagePath = path.normalize(cachePackageEntry.root.path);
final packageDirectory = Directory(packagePath);
if (!packageDirectory.existsSync()) {
final errorMessage =
'''[$dependencyName] Could not find package directory at $packagePath.''';
if (!ignoreFailures) {
progress.cancel();
_logger.err(errorMessage);
return ExitCode.noInput.code;
}

_logger.err('\n$errorMessage');
licenses[dependencyName] = {SpdxLicense.$unknown.value};
continue;
}

final licenseFile = File(path.join(packagePath, 'LICENSE'));
if (!licenseFile.existsSync()) {
licenses[dependencyName] = {SpdxLicense.$unknown.value};
continue;
}

final licenseFileContent = licenseFile.readAsStringSync();

late final detector.Result detectorResult;
try {
detectorResult =
await detectLicense(licenseFileContent, _defaultDetectionThreshold);
} catch (e) {
final errorMessage =
'[$dependencyName] Unexpected failure with error: $e';
'''[$dependencyName] Failed to detect license from $packagePath: $e''';
if (!ignoreFailures) {
progress.cancel();
_logger.err(errorMessage);
return ExitCode.software.code;
}

_logger.err('\n$errorMessage');
} finally {
licenses[dependencyName] = rawLicense;
licenses[dependencyName] = {SpdxLicense.$unknown.value};
continue;
}

final rawLicense = detectorResult.matches
// ignore: invalid_use_of_visible_for_testing_member
.map((match) => match.license.identifier)
.toSet();
licenses[dependencyName] = rawLicense;
}

late final _BannedDependencyLicenseMap? bannedDependencies;
Expand Down Expand Up @@ -246,6 +330,23 @@ PubspecLock? _tryParsePubspecLock(File pubspecLockFile) {
}
}

/// Attempts to find a [package_config.PackageConfig] using
/// [package_config.findPackageConfig].
///
/// If [package_config.findPackageConfig] fails to find a package config `null`
/// is returned.
Future<package_config.PackageConfig?> _tryFindPackageConfig(
Directory directory,
) async {
try {
final findPackageConfig =
findPackageConfigOverride ?? package_config.findPackageConfig;
return await findPackageConfig(directory);
} catch (error) {
return null;
}
}

/// Verifies that all [licenses] are valid license inputs.
///
/// Valid license inputs are:
Expand Down
5 changes: 2 additions & 3 deletions lib/src/commands/packages/packages.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ import 'package:args/command_runner.dart';
import 'package:mason/mason.dart';
import 'package:very_good_cli/src/commands/packages/commands/check/check.dart';
import 'package:very_good_cli/src/commands/packages/commands/commands.dart';
import 'package:very_good_cli/src/pub_license/pub_license.dart';

/// {@template packages_command}
/// `very_good packages` command for managing packages.
/// {@endtemplate}
class PackagesCommand extends Command<int> {
/// {@macro packages_command}
PackagesCommand({Logger? logger, PubLicense? pubLicense}) {
PackagesCommand({Logger? logger}) {
addSubcommand(PackagesGetCommand(logger: logger));
addSubcommand(PackagesCheckCommand(logger: logger, pubLicense: pubLicense));
addSubcommand(PackagesCheckCommand(logger: logger));
}

@override
Expand Down
Loading