[dart2js] Add a couple scripts to aid our migration.

These are simple scripts to count progress and find migration candidates.

Change-Id: I872d85891001349dadbcf1d67e64ab5aa993d2a5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/250146
Reviewed-by: Stephen Adams <sra@google.com>
Commit-Queue: Sigmund Cherem <sigmund@google.com>
Reviewed-by: Mayank Patke <fishythefish@google.com>

pkg/compiler/pubspec.yaml

@@ -34,3 +34,4 @@ dev_dependencies:
   modular_test: any
   sourcemap_testing: any
   testing: any
+  vm: any

pkg/compiler/tool/null_safety/candidates.dart

@@ -0,0 +1,152 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// Script to identify good opportunities for null safety migration.
+///
+/// This script sorts libraries based on a "migratable" order. We compute
+/// this order by counting how many of a library's dependencies have been
+/// migrated.
+
+import 'dart:io';
+
+import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
+import 'package:_fe_analyzer_shared/src/parser/parser.dart';
+import 'package:_fe_analyzer_shared/src/scanner/io.dart'
+    show readBytesFromFileSync;
+import 'package:_fe_analyzer_shared/src/scanner/scanner.dart';
+import 'package:front_end/src/api_prototype/front_end.dart';
+import 'package:front_end/src/api_prototype/language_version.dart';
+import 'package:front_end/src/api_prototype/terminal_color_support.dart'
+    show printDiagnosticMessage;
+import 'package:front_end/src/base/processed_options.dart';
+import 'package:front_end/src/fasta/compiler_context.dart';
+import 'package:front_end/src/fasta/source/diet_parser.dart';
+import 'package:front_end/src/fasta/source/directive_listener.dart';
+import 'package:front_end/src/fasta/uri_translator.dart' show UriTranslator;
+import 'package:kernel/target/targets.dart' show TargetFlags;
+import 'package:vm/target/vm.dart' show VmTarget;
+
+void main(List<String> args) async {
+  var prefix = args.isEmpty ? 'pkg/compiler/' : args.first;
+  var files = <Uri, List<int>>{};
+  var isLegacy = <Uri>{};
+  var isNullSafe = <Uri>{};
+
+  var entryUri = Uri.parse('package:compiler/src/dart2js.dart');
+  var options = CompilerOptions()
+    ..sdkRoot = Uri.base.resolve("sdk/")
+    ..onDiagnostic = _onDiagnosticMessageHandler
+    ..compileSdk = true
+    ..packagesFileUri = Uri.base.resolve('.dart_tool/package_config.json')
+    ..target = VmTarget(TargetFlags());
+  var pOptions = ProcessedOptions(options: options);
+  var uriResolver = await pOptions.getUriTranslator();
+  var context = CompilerContext(pOptions);
+  await context.runInContext((_) async {
+    collectSources(uriResolver, entryUri, files);
+  });
+
+  for (var file in files.keys) {
+    if (await uriUsesLegacyLanguageVersion(file, options)) {
+      isLegacy.add(file);
+    } else {
+      isNullSafe.add(file);
+    }
+  }
+
+  var fileSummary = <Uri, FileData>{};
+  for (var file in files.keys) {
+    if (!file.path.contains(prefix)) continue;
+    var directives = extractDirectiveUris(files[file]!)
+        .map(file.resolve)
+        .where((uri) => uri.path.contains('pkg/compiler/'));
+    var migrated = directives.where(isNullSafe.contains).length;
+    var total = directives.length;
+    fileSummary[file] = FileData(isNullSafe.contains(file), total, migrated);
+  }
+
+  var keys = fileSummary.keys.toList();
+  keys.sort((a, b) {
+    var fa = fileSummary[a]!;
+    var fb = fileSummary[b]!;
+    if (fa.isNullSafe && !fb.isNullSafe) return -1;
+    if (fb.isNullSafe && !fa.isNullSafe) return 1;
+    if (fa.totalDependencies == 0 && fb.totalDependencies != 0) return -1;
+    if (fb.totalDependencies == 0 && fa.totalDependencies != 0) return 1;
+    if (fa.ratio != fb.ratio) return fb.ratio.compareTo(fa.ratio);
+    return fb.migratedDependencies.compareTo(fb.migratedDependencies);
+  });
+
+  for (var file in keys) {
+    var data = fileSummary[file]!;
+    String status;
+    String text = shorten(file);
+    if (data.isNullSafe) {
+      status = '\x1b[33mmigrated ---\x1b[0m | $text';
+    } else if (data.totalDependencies == 0) {
+      status = '\x1b[32mready    ---\x1b[0m | $text';
+    } else if (data.ratio == 1.0) {
+      status = '\x1b[32mready   100%\x1b[0m | $text';
+    } else {
+      var perc = (data.ratio * 100).toStringAsFixed(0).padLeft(3);
+      status = '\x1b[31mwait    $perc%\x1b[0m'
+          ' | $text [${data.migratedDependencies} / ${data.totalDependencies}]';
+    }
+    print(status);
+  }
+}
+
+class FileData {
+  final bool isNullSafe;
+  final int totalDependencies;
+  final int migratedDependencies;
+
+  double get ratio => migratedDependencies / totalDependencies;
+  FileData(this.isNullSafe, this.totalDependencies, this.migratedDependencies);
+}
+
+void _onDiagnosticMessageHandler(DiagnosticMessage m) {
+  if (m.severity == Severity.internalProblem || m.severity == Severity.error) {
+    printDiagnosticMessage(m, stderr.writeln);
+    exitCode = 1;
+  }
+}
+
+/// Add to [files] all sources reachable from [start].
+void collectSources(
+    UriTranslator uriResolver, Uri start, Map<Uri, List<int>> files) {
+  void helper(Uri uri) {
+    if (uri.scheme == 'dart') return;
+    uri = uriResolver.translate(uri) ?? uri;
+    if (!uri.path.contains('pkg/compiler/')) return;
+    if (files.containsKey(uri)) return;
+    var contents = readBytesFromFileSync(uri);
+    files[uri] = contents;
+    for (var directiveUri in extractDirectiveUris(contents)) {
+      helper(uri.resolve(directiveUri));
+    }
+  }
+
+  helper(start);
+}
+
+/// Parse [contents] as a Dart program and return the URIs that appear in its
+/// import, export, and part directives.
+Set<String> extractDirectiveUris(List<int> contents) {
+  var listener = new DirectiveListener();
+  new TopLevelParser(listener,
+          useImplicitCreationExpression: useImplicitCreationExpressionInCfe)
+      .parseUnit(scan(contents).tokens);
+  // Note: this purposely ignores part files (listener.parts).
+  return new Set<String>()
+    ..addAll(listener.imports.map((directive) => directive.uri!))
+    ..addAll(listener.exports.map((directive) => directive.uri!));
+}
+
+String shorten(Uri uri) {
+  if (uri.scheme != 'file') return uri.toString();
+  final prefix = Uri.base.path;
+  if (uri.path.startsWith(prefix)) return uri.path.substring(prefix.length);
+  return uri.toString();
+}

pkg/compiler/tool/null_safety/tally.dart

@@ -0,0 +1,91 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// Script to count progress of the pkg/compiler/lib/ migration
+
+import 'dart:io';
+
+void main(List<String> args) {
+  var path = args.isEmpty ? 'pkg/compiler/lib/' : args.first;
+  var dart2jsDir = Directory.fromUri(Uri.base.resolve(path));
+  var entries = <FileData>[];
+  for (var file in dart2jsDir.listSync(recursive: true)) {
+    if (file is File && file.uri.path.endsWith('.dart')) {
+      entries.add(FileData(file));
+    }
+  }
+
+  var tally = Tally();
+  for (var e in entries) {
+    tally.totalFiles++;
+    tally.totalBytes += e.sizeBytes;
+    tally.totalLOC += e.sizeLOC;
+    if (e.nullSafe) {
+      tally.migratedFiles++;
+      tally.migratedBytes += e.sizeBytes;
+      tally.migratedLOC += e.sizeLOC;
+    }
+  }
+
+  print(tally.formatString());
+  //print(tally.csvRow());
+}
+
+/// Details about each file in the package to properly count migration progress.
+class FileData {
+  final Uri path;
+  final int sizeBytes;
+  final int sizeLOC;
+  final bool nullSafe;
+
+  FileData._(this.path, this.sizeBytes, this.sizeLOC, this.nullSafe);
+
+  factory FileData(File file) {
+    var contents = file.readAsStringSync();
+    var length = contents.length;
+    var sizeLOC = '\n'.allMatches(contents).length;
+    var nullSafe = !contents.contains("// @dart = 2.10");
+    return FileData._(file.uri, length, sizeLOC, nullSafe);
+  }
+}
+
+/// Cumulative information about the status of the null safety migration.
+class Tally {
+  int totalFiles = 0;
+  int migratedFiles = 0;
+  int totalBytes = 0;
+  int migratedBytes = 0;
+  int totalLOC = 0;
+  int migratedLOC = 0;
+
+  /// Emit a readable table representation of the null safety progress.
+  String formatString() {
+    String _pad(String text, int width) {
+      return (' ' * (10 - text.length)) + text;
+    }
+
+    String _row(String name, int a, int b) {
+      var padA = _pad('$a', 10);
+      var padB = _pad('$b', 10);
+      var padC = _pad((a * 100 / b).toStringAsFixed(2), 10);
+      return '${_pad(name, 8)} $padA $padB $padC%';
+    }
+
+    return '${_pad("", 10)} ${_pad("migrated", 10)} ${_pad("total", 10)} ${_pad("%", 10)}\n'
+        '${_row('Files', migratedFiles, totalFiles)}\n'
+        '${_row('Lines', migratedLOC, totalLOC)}\n'
+        '${_row('Bytes', migratedBytes, totalBytes)}';
+  }
+
+  /// Emit a csv representation of the null safety progress, useful to track
+  /// data over time.
+  String csvRow() => [
+        totalFiles,
+        migratedFiles,
+        totalBytes,
+        migratedBytes,
+        totalLOC,
+        migratedLOC,
+      ].join(',');
+}