Skip to content

Modular analyzer, added end2end test. #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 26, 2017
Merged

Modular analyzer, added end2end test. #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions bin/main.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ main(List<String> arguments) async {

try {
try {
var summary =
await inspectPackage(pkg, version: version, pubCachePath: tempPath);
PackageAnalyzer analyzer = new PackageAnalyzer(pubCacheDir: tempPath);
var summary = await analyzer.inspectPackage(pkg, version: version);

print(prettyJson(summary));
} catch (e, stack) {
Expand Down
260 changes: 68 additions & 192 deletions lib/pana.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,219 +6,95 @@ import 'dart:collection';
import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as p;
import 'package:pub_semver/pub_semver.dart';

import 'src/analyzer_output.dart';
import 'src/logging.dart';
import 'src/pub_summary.dart';
import 'src/sdk_env.dart';
import 'src/summary.dart';
import 'src/utils.dart';

export 'src/pub_summary.dart';
export 'src/summary.dart';
export 'src/utils.dart';

String prettyJson(obj) => const JsonEncoder.withIndent(' ').convert(obj).trim();
class PackageAnalyzer {
DartSdk _dartSdk;
PubEnvironment _pubEnv;

Future<Summary> inspectPackage(String pkgName,
{String version, String pubCachePath}) async {
var versionResult = _handleErrors(await Process.run('dart', ['--version']));

var sdkVersion = versionResult.stderr.toString().trim();
log.info("SDK: $sdkVersion");
PackageAnalyzer({String sdkDir, String pubCacheDir}) {
_dartSdk = new DartSdk(sdkDir: sdkDir);
_pubEnv = new PubEnvironment(dartSdk: _dartSdk, pubCacheDir: pubCacheDir);
}

log.info("Package: $pkgName");
Future<Summary> inspectPackage(String package, {String version}) async {
var sdkVersion = await _dartSdk.version;
log.info("SDK: $sdkVersion");

Version ver;
if (version != null) {
ver = new Version.parse(version);
log.info("Version: $ver");
}
log.info("Package: $package");

if (pubCachePath != null) {
log.info("Using .package-cache: ${pubCachePath}");
var tempDir = new Directory(pubCachePath).resolveSymbolicLinksSync();
if (tempDir != pubCachePath) {
throw new ArgumentError([
"Make sure you resolve symlinks:",
pubCachePath,
tempDir
].join('\n'));
Version ver;
if (version != null) {
ver = new Version.parse(version);
log.info("Version: $ver");
}
}

log.info("Downloading package...");
var pkgDir =
await downloadPkg(pkgName, version: ver, pubCachePath: pubCachePath);
log.info("Package at ${pkgDir.path}");

log.info('Counting files...');
var result = _handleErrors(
await Process.run('find', [pkgDir.path, '-name', '*.dart']));

var dartFiles = new SplayTreeSet<String>.from(
LineSplitter.split(result.stdout).map((path) {
assert(p.isWithin(pkgDir.path, path));

return p.relative(path, from: pkgDir.path);
}));

log.info("Checking formatting...");
var unformattedFiles =
new SplayTreeSet<String>.from(filesNeedingFormat(pkgDir.path));

log.info("Pub upgrade...");
var summary = await pubUpgrade(pkgDir.path, pubCachePath: pubCachePath);
log.info("Package version: ${summary.pkgVersion}");

Set<AnalyzerOutput> analyzerItems;
try {
analyzerItems = await pkgAnalyze(pkgDir.path);
} on ArgumentError catch (e) {
if (e.toString().contains("No dart files found at: .")) {
log.warning("No files to analyze...");
analyzerItems = new Set<AnalyzerOutput>();
} else {
rethrow;
if (_pubEnv.pubCacheDir != null) {
log.info("Using .package-cache: ${_pubEnv.pubCacheDir}");
}
}

return new Summary(sdkVersion, pkgName, pkgDir.version, dartFiles, summary,
analyzerItems, unformattedFiles);
}

List<String> filesNeedingFormat(String pkgPath) {
var result = Process
.runSync('dartfmt', ['--dry-run', '--set-exit-if-changed', pkgPath]);

if (result.exitCode == 0) {
return const [];
}

var lines = LineSplitter.split(result.stdout).toList()..sort();

assert(lines.isNotEmpty);

return lines;
}

Future<Set<AnalyzerOutput>> pkgAnalyze(String pkgPath) async {
log.info('Running `dartanalyzer`...');
var proc = await Process.run(
'dartanalyzer', ['--strong', '--format', 'machine', '.'],
workingDirectory: pkgPath);

try {
return new SplayTreeSet.from(LineSplitter
.split(proc.stderr)
.map((s) => AnalyzerOutput.parse(s, projectDir: pkgPath))
.where((e) => e != null));
} on ArgumentError {
// TODO: we should figure out a way to succeed here, right?
// Or at least do partial results and not blow up
log.severe("Bad input?");
log.severe(proc.stderr);
rethrow;
}
}

Future<PubSummary> pubUpgrade(String pkgPath, {String pubCachePath}) async {
var pubEnv = new Map<String, String>.from(_pubEnv);
if (pubCachePath != null) {
pubEnv['PUB_CACHE'] = pubCachePath;
}

var retryCount = 3;
ProcessResult result;
do {
retryCount--;
log.info('Running `pub upgrade`...');
result = await Process.run('pub', ['upgrade', '--verbosity', 'all'],
workingDirectory: pkgPath, environment: pubEnv);
} while (result.exitCode != 0 && retryCount > 0);
return PubSummary.create(
result.exitCode, result.stdout, result.stderr, pkgPath);
}

Future<PkgInstallInfo> downloadPkg(String pkgName,
{Version version, String pubCachePath}) async {
var args = ['cache', 'add', '--verbose'];

if (version != null) {
args.addAll(['--version', version.toString()]);
}

args.add(pkgName);

var pubEnv = new Map<String, String>.from(_pubEnv);
if (pubCachePath != null) {
pubEnv['PUB_CACHE'] = pubCachePath;
}

var result =
_handleErrors(await Process.run('pub', args, environment: pubEnv));

var match = _versionDownloadRexexp.allMatches(result.stdout.trim()).single;

var pkgMatch = match[1];
assert(pkgMatch == pkgName);

var versionString = match[2];
assert(versionString.endsWith('.'));
while (versionString.endsWith('.')) {
versionString = versionString.substring(0, versionString.length - 1);
}

var downloadedVersion = new Version.parse(versionString);

if (version != null) {
assert(downloadedVersion == version);
}

// now get all installed packages
result = _handleErrors(
await Process.run('pub', ['cache', 'list'], environment: pubEnv));

var json = JSON.decode(result.stdout) as Map;

var location = json['packages'][pkgName][versionString]['location'] as String;
log.info("Downloading package...");
PackageLocation pkgInfo =
await _pubEnv.getLocation(package, version: ver?.toString());
String pkgDir = pkgInfo.location;
log.info("Package at $pkgDir");

log.info('Counting files...');
var dartFiles = new SplayTreeSet<String>.from(
await listFiles(pkgDir, endsWith: '.dart'));

log.info("Checking formatting...");
var unformattedFiles = new SplayTreeSet<String>.from(
await _dartSdk.filesNeedingFormat(pkgDir));

log.info("Pub upgrade...");
ProcessResult upgrade = await _pubEnv.runUpgrade(pkgDir);
var summary = PubSummary.create(
upgrade.exitCode, upgrade.stdout, upgrade.stderr, pkgDir);
log.info("Package version: ${summary.pkgVersion}");

Set<AnalyzerOutput> analyzerItems;
try {
analyzerItems = await _pkgAnalyze(pkgDir);
} on ArgumentError catch (e) {
if (e.toString().contains("No dart files found at: .")) {
log.warning("No files to analyze...");
analyzerItems = new Set<AnalyzerOutput>();
} else {
rethrow;
}
}

if (location == null) {
throw "Huh? This should be cached!";
return new Summary(sdkVersion, package, new Version.parse(pkgInfo.version),
dartFiles, summary, analyzerItems, unformattedFiles);
}

return new PkgInstallInfo(pkgName, downloadedVersion, location);
}

class PkgInstallInfo {
final String name;
final Version version;
final String path;

PkgInstallInfo(this.name, this.version, this.path);
}

final _versionDownloadRexexp =
new RegExp(r"^MSG : (?:Downloading |Already cached )([\w-]+) (.+)$");

const _pubEnv = const <String, String>{'PUB_ENVIRONMENT': 'kevmoo.pkg_clean'};

ProcessResult _handleErrors(ProcessResult result) {
if (result.exitCode != 0) {
if (result.exitCode == 69) {
// could be a pub error. Let's try to parse!
var lines = LineSplitter
.split(result.stderr)
.where((l) => l.startsWith("ERR "))
.join('\n');
if (lines.isNotEmpty) {
throw lines;
}
Future<Set<AnalyzerOutput>> _pkgAnalyze(String pkgPath) async {
log.info('Running `dartanalyzer`...');
var proc = await _dartSdk.runAnalyzer(pkgPath);

try {
return new SplayTreeSet.from(LineSplitter
.split(proc.stderr)
.map((s) => AnalyzerOutput.parse(s, projectDir: pkgPath))
.where((e) => e != null));
} on ArgumentError {
// TODO: we should figure out a way to succeed here, right?
// Or at least do partial results and not blow up
log.severe("Bad input?");
log.severe(proc.stderr);
rethrow;
}

throw "Problem running proc: exit code - " +
[result.exitCode, result.stdout, result.stderr]
.map((e) => e.toString().trim())
.join('<***>');
}
return result;
}
32 changes: 3 additions & 29 deletions lib/src/pub_summary.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,38 +4,12 @@ import 'dart:io';

import 'package:path/path.dart' as p;
import 'package:pub_semver/pub_semver.dart';
import 'package:yaml/yaml.dart';

import 'utils.dart';

final _prefix = new RegExp(r"(MSG| ) (:|\|) (?:\+| ) (.+)");
final _infoRegexp = new RegExp(r"(\w+) (\S+)(?: \((\S+) available\))?");

_toSortedMap(Object item) {
if (item is Map) {
return new SplayTreeMap<String, Object>.fromIterable(item.keys,
value: (k) => _toSortedMap(item[k]));
} else if (item is List) {
return item.map(_toSortedMap).toList();
} else {
return item;
}
}

Object sortedJson(obj) {
var fullJson = JSON.decode(JSON.encode(obj));
return _toSortedMap(fullJson);
}

Map<String, Object> _yamlToJson(String pubspecContent) {
if (pubspecContent == null) {
return null;
}
var yamlMap = loadYaml(pubspecContent) as YamlMap;

// A bit paranoid, but I want to make sure this is valid JSON before we got to
// the encode phase.
return sortedJson(JSON.decode(JSON.encode(yamlMap)));
}

class PubSummary {
final int exitCode;
final String stdoutValue;
Expand All @@ -53,7 +27,7 @@ class PubSummary {
this.availableVersions,
String pubspecContent,
this.lockFileContent)
: pubspec = _yamlToJson(pubspecContent);
: pubspec = yamlToJson(pubspecContent);

static PubSummary create(
int exitCode, String procStdout, String procStderr, String path) {
Expand Down
Loading