FEDX-2972: Pubspec Indexing

.github/workflows/ci.yaml

@@ -44,10 +44,10 @@ jobs:
 
       - name: Snapshots Diff Check
         run: |
-          dart run scip_dart ./snapshots/input/basic-project
+          dart run scip_dart ./snapshots/input/basic-project --index-pubspec
           ./scip snapshot --to ./snapshots/output/basic-project
 
-          dart run scip_dart ./snapshots/input/diagnostics
+          dart run scip_dart ./snapshots/input/diagnostics --index-pubspec
           ./scip snapshot --to ./snapshots/output/diagnostics
 
           if [[ -z "$(git status --porcelain ./snapshots/output)" ]];

Makefile

@@ -1,8 +1,8 @@
 regen-snapshots:
-	dart run scip_dart ./snapshots/input/basic-project
+	dart run scip_dart ./snapshots/input/basic-project --index-pubspec
 	scip snapshot --to ./snapshots/output/basic-project
 
-	dart run scip_dart ./snapshots/input/diagnostics
+	dart run scip_dart ./snapshots/input/diagnostics --index-pubspec
 	scip snapshot --to ./snapshots/output/diagnostics
 
 run:

bin/scip_dart.dart

@@ -1,11 +1,13 @@
 import 'dart:io';
 
 import 'package:args/args.dart';
+import 'package:pubspec_lock_parse/pubspec_lock_parse.dart';
 import 'package:scip_dart/scip_dart.dart';
 import 'package:package_config/package_config.dart';
 import 'package:pubspec_parse/pubspec_parse.dart';
 import 'package:path/path.dart' as p;
 import 'package:scip_dart/src/flags.dart';
+import 'package:scip_dart/src/pubspec_indexer.dart';
 import 'package:scip_dart/src/version.dart';
 
 Future<void> main(List<String> args) async {
@@ -17,6 +19,11 @@ Future<void> main(List<String> args) async {
           help:
               'The output file to write the index to. Use "-" to write to stdout',
         )
+        ..addFlag(
+          'index-pubspec',
+          defaultsTo: false,
+          help: 'Whether or not to index the pubspec.yaml file',
+        )
         ..addFlag(
           'performance',
           aliases: ['perf'],
@@ -74,10 +81,27 @@ Future<void> main(List<String> args) async {
     stderr.writeln('ERROR: Unable to locate pubspec.yaml');
     exit(1);
   }
-  final pubspec = Pubspec.parse(pubspecFile.readAsStringSync());
+  final pubspecStr = pubspecFile.readAsStringSync();
+  final pubspec = Pubspec.parse(pubspecStr);
 
   final index = await indexPackage(packageRoot, packageConfig, pubspec);
 
+  if (result['index-pubspec'] as bool) {
+    final pubspecLockFile = File(p.join(packageRoot, 'pubspec.lock'));
+    if (!pubspecLockFile.existsSync()) {
+      stderr.writeln(
+          'ERROR: Unable to locate pubspec.lock. Have you ran pub get?');
+      exit(1);
+    }
+    final pubspecLock = PubspecLock.parse(pubspecLockFile.readAsStringSync());
+
+    index.documents.add(indexPubspec(
+      pubspec: pubspec,
+      pubspecStr: pubspecStr,
+      pubspecLock: pubspecLock,
+    ));
+  }
+
   if (result['output'] as String == '-') {
     stdout.add(index.writeToBuffer());
     stdout.flush();

lib/src/indexer.dart

@@ -1,5 +1,3 @@
-import 'dart:io';
-
 import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
 import 'package:analyzer/dart/analysis/results.dart';
 import 'package:path/path.dart' as p;
@@ -19,17 +17,6 @@ Future<Index> indexPackage(
 ) async {
   final dirPath = p.normalize(p.absolute(root));
 
-  final rootHasPubspecFile = await File(
-    p.join(dirPath, 'pubspec.yaml'),
-  ).exists();
-  if (!rootHasPubspecFile) {
-    stderr.writeln(
-      'Provided path does not contain a pubspec.yaml file. '
-      'Unable to index',
-    );
-    exit(1);
-  }
-
   final metadata = Metadata(
     projectRoot: Uri.file(dirPath).toString(),
     textDocumentEncoding: TextEncoding.UTF8,

lib/src/pubspec_indexer.dart

@@ -0,0 +1,126 @@
+import 'package:analyzer/source/line_info.dart';
+import 'package:pubspec_lock_parse/pubspec_lock_parse.dart';
+import 'package:pubspec_parse/pubspec_parse.dart';
+import 'package:scip_dart/src/utils.dart';
+
+import 'gen/scip.pb.dart';
+
+/// Generates a scip document representation of a pubspec.yaml file. Output
+/// specification looks like the following:
+///
+/// ```yaml
+/// # scip-dart pub foo_pkg 1.0.0 `pubspec.yaml`/
+///  name: foo_pkg
+///  version: 1.0.0
+///
+///  dependencies:
+///    some_pkg: ^1.0.5
+/// #  ^^^^^^^^ scip-dart pub some_pkg 1.5.6 `pubspec.yaml`/
+/// ```
+Document indexPubspec({
+  required Pubspec pubspec,
+  required String pubspecStr,
+  required PubspecLock pubspecLock,
+}) {
+  final pubspecLineInfo = LineInfo.fromContent(pubspecStr);
+
+  final fileSymbol = [
+    'scip-dart',
+    'pub',
+    pubspec.name,
+    pubspec.version!,
+    '`pubspec.yaml`/',
+  ].join(' ');
+
+  final symbols = <SymbolInformation>[
+    SymbolInformation(
+        symbol: fileSymbol,
+        kind: SymbolInformation_Kind
+            .UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency
+        signatureDocumentation: Document(
+            language: Language.YAML.name,
+            text: ['name: ${pubspec.name}', 'version: ${pubspec.version}']
+                .join('\n')))
+  ];
+  final occurrences = <Occurrence>[
+    Occurrence(
+      symbol: fileSymbol,
+      range: [0, 0, 0],
+    )
+  ];
+
+  final deps = {
+    DependencyKind.regular: pubspec.dependencies,
+    DependencyKind.dev: pubspec.devDependencies,
+    DependencyKind.override: pubspec.dependencyOverrides,
+  };
+  for (final kind in deps.keys) {
+    for (final dep in deps[kind]!.entries) {
+      final info = _buildSymbol(dep.key, pubspecLock);
+      symbols.add(info);
+      occurrences.add(Occurrence(
+        symbol: info.symbol,
+        range: pubspecLineInfo.getDependencyRange(pubspecStr, dep.key, kind),
+      ));
+    }
+  }
+
+  return Document(
+    language: Language.YAML.name,
+    relativePath: 'pubspec.yaml',
+    symbols: symbols,
+    occurrences: occurrences,
+  );
+}
+
+SymbolInformation _buildSymbol(String depName, PubspecLock lock) {
+  final depVersion = lock.packages[depName]?.version.toString();
+  if (depVersion == null) {
+    throw Exception(
+        'Unable to find ${depName} in pubspec.lock. Have you ran pub get?');
+  }
+
+  return SymbolInformation(
+      displayName: depName,
+      symbol: [
+        'scip-dart',
+        'pub',
+        depName,
+        depVersion,
+        '`pubspec.yaml`/',
+      ].join(' '),
+      kind: SymbolInformation_Kind
+          .UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency
+      signatureDocumentation: Document(
+          language: Language.YAML.name,
+          text: [
+            'name: $depName',
+            'version: $depVersion',
+          ].join('\n')));
+}
+
+enum DependencyKind {
+  regular('^dependencies:'),
+  dev('^dev_dependencies:'),
+  override('^dependency_overrides:');
+
+  final String matcher;
+  const DependencyKind(this.matcher);
+}
+
+extension _PubspecLineInfo on LineInfo {
+  List<int> getDependencyRange(
+      String pubspecStr, String name, DependencyKind kind) {
+    final section =
+        getYamlSection(pubspecStr, RegExp(kind.matcher, multiLine: true));
+    if (section == null) {
+      throw Exception(
+          'Unable to find section "${kind.matcher}" in pubspec.yaml');
+    }
+    final sectionStart = pubspecStr.indexOf(section);
+    final depStart = section.indexOf('  ${name}:') + 2;
+    final startOffset = sectionStart + depStart;
+
+    return getRange(startOffset, name.length);
+  }
+}

lib/src/utils.dart

@@ -3,6 +3,7 @@ import 'dart:io';
 import 'package:analyzer/source/line_info.dart';
 import 'package:path/path.dart' as p;
 
+import 'package:collection/collection.dart';
 import 'package:scip_dart/src/flags.dart';
 
 /// Returns a list of all the pubspec.yaml paths under a directory.
@@ -41,3 +42,46 @@ extension LineInfoExtension on LineInfo {
     ];
   }
 }
+
+String? getYamlSection(
+  String str,
+  Pattern startRegex, {
+  bool skipMatchedLine = false,
+}) {
+  final indentSize = getYamlIndentSize(str);
+
+  final match = startRegex.allMatches(str).firstOrNull;
+  if (match == null) return null;
+
+  final startingCharacter = match.start;
+
+  final sectionAndAfterLines = str.substring(startingCharacter).split('\n');
+  final sectionKeyLine = sectionAndAfterLines[0];
+  final sectionKeyIndentSize =
+      sectionKeyLine.length - sectionKeyLine.trimLeft().length;
+
+  final inSectionIndent =
+      List.filled(sectionKeyIndentSize + indentSize, ' ').join();
+
+  final inSectionLines = sectionAndAfterLines
+      .skip(1) // skip the section key line
+      .takeWhile(
+          (line) => line.trim().isEmpty || line.startsWith(inSectionIndent))
+      .toList();
+
+  return [if (!skipMatchedLine) sectionKeyLine, ...inSectionLines]
+      .where((line) => line.trim().isNotEmpty)
+      .join('\n');
+}
+
+int getYamlIndentSize(String str) {
+  final lines = str.split('\n');
+
+  for (var line in lines) {
+    if (line.startsWith(' ')) {
+      return line.length - line.trimLeft().length;
+    }
+  }
+
+  return 0;
+}

pubspec.lock

@@ -58,7 +58,7 @@ packages:
     source: hosted
     version: "2.0.3"
   collection:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: collection
       sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a
@@ -185,6 +185,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.4"
+  pubspec_lock_parse:
+    dependency: "direct main"
+    description:
+      name: pubspec_lock_parse
+      sha256: "020cb470287124c936c30ebfc2f927b287f275b7bf7fc2ab11577e592c017764"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.2.0"
   pubspec_parse:
     dependency: "direct main"
     description:
@@ -258,4 +266,4 @@ packages:
     source: hosted
     version: "3.1.2"
 sdks:
-  dart: ">=2.19.0 <3.0.0"
+  dart: ">=2.19.6 <3.0.0"

pubspec.yaml

@@ -4,7 +4,7 @@ description: generates scip bindings for dart files
 repository: https://github.com/Workiva/scip-dart
 
 environment:
-  sdk: ">=2.19.0 <3.0.0"
+  sdk: ">=2.19.6 <3.0.0"
 
 executables:
   scip_dart:
@@ -16,6 +16,8 @@ dependencies:
   path: ^1.8.3
   protobuf: ^3.1.0
   pubspec_parse: ^1.2.1
+  pubspec_lock_parse: ^2.2.0
+  collection: ^1.18.0
 
 dev_dependencies:
   chalk: ^1.2.1