From 259194bb54371793fc611cf1b9aa4c75e3e8dda3 Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:23:54 -0600 Subject: [PATCH 1/6] implemented spike of pubspec indexing --- bin/scip_dart.dart | 14 ++- lib/src/indexer.dart | 115 ++++++++++++++++-- lib/src/utils.dart | 44 +++++++ pubspec.lock | 24 ++++ pubspec.yaml | 3 + snapshots/input/staging-project/pubspec.lock | 100 ++++++++++++++- snapshots/input/staging-project/pubspec.yaml | 4 + snapshots/output/basic-project/pubspec.yaml | 10 ++ snapshots/output/diagnostics/pubspec.yaml | 10 ++ snapshots/output/staging-project/pubspec.yaml | 11 ++ 10 files changed, 318 insertions(+), 17 deletions(-) create mode 100755 snapshots/output/basic-project/pubspec.yaml create mode 100755 snapshots/output/diagnostics/pubspec.yaml create mode 100755 snapshots/output/staging-project/pubspec.yaml diff --git a/bin/scip_dart.dart b/bin/scip_dart.dart index 0fa043a9..3b0e2b4b 100644 --- a/bin/scip_dart.dart +++ b/bin/scip_dart.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:args/args.dart'; +import 'package:pubspec_lock_parse/pubspec_lock_parse.dart'; import 'package:scip_dart/scip_dart.dart'; import 'package:package_config/package_config.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; @@ -74,9 +75,18 @@ Future main(List args) async { stderr.writeln('ERROR: Unable to locate pubspec.yaml'); exit(1); } - final pubspec = Pubspec.parse(pubspecFile.readAsStringSync()); + final pubspecStr = pubspecFile.readAsStringSync(); + final pubspec = Pubspec.parse(pubspecStr); - final index = await indexPackage(packageRoot, packageConfig, pubspec); + final pubspecLockFile = File(p.join(packageRoot, 'pubspec.lock')); + if (!pubspecLockFile.existsSync()) { + stderr.writeln('ERROR: Unable to locate pubspec.lock. Have you ran pub get?'); + exit(1); + } + final pubspecLock = PubspecLock.parse(pubspecLockFile.readAsStringSync()); + + + final index = await indexPackage(packageRoot, packageConfig, pubspec, pubspecStr, pubspecLock); if (result['output'] as String == '-') { stdout.add(index.writeToBuffer()); diff --git a/lib/src/indexer.dart b/lib/src/indexer.dart index 6cbcaaaf..e7648938 100644 --- a/lib/src/indexer.dart +++ b/lib/src/indexer.dart @@ -1,9 +1,9 @@ -import 'dart:io'; - import 'package:analyzer/dart/analysis/analysis_context_collection.dart'; import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/source/line_info.dart'; import 'package:path/path.dart' as p; import 'package:package_config/package_config.dart'; +import 'package:pubspec_lock_parse/pubspec_lock_parse.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; import 'package:scip_dart/src/flags.dart'; @@ -16,20 +16,11 @@ Future indexPackage( String root, PackageConfig packageConfig, Pubspec pubspec, + String pubspecStr, + PubspecLock pubspecLock, ) async { final dirPath = p.normalize(p.absolute(root)); - final rootHasPubspecFile = await File( - p.join(dirPath, 'pubspec.yaml'), - ).exists(); - if (!rootHasPubspecFile) { - stderr.writeln( - 'Provided path does not contain a pubspec.yaml file. ' - 'Unable to index', - ); - exit(1); - } - final metadata = Metadata( projectRoot: Uri.file(dirPath).toString(), textDocumentEncoding: TextEncoding.UTF8, @@ -103,9 +94,107 @@ Future indexPackage( print('Parsing Ast took: ${st.elapsedMilliseconds}ms'); } + documents.add(_indexPubspec(root, pubspec, pubspecStr, pubspecLock)); + return Index( metadata: metadata, documents: documents, externalSymbols: globalExternalSymbols, ); } + + +Document _indexPubspec(String rootPath, Pubspec pubspec, String pubspecStr, PubspecLock lock) { + final pubspecLineInfo = LineInfo.fromContent(pubspecStr); + + SymbolInformation __buildSymbol(String depName, Dependency dep) { + final depVersion = lock.packages[depName]!.version.toString(); + + return SymbolInformation( + displayName: depName, + symbol: [ + 'scip-dart', + 'pub', + depName, + depVersion, + '`pubspec.yaml`/', + ].join(' '), + kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: [ + 'name: $depName', + 'version: $depVersion', + ].join('\n') + ) + ); + } + + Occurrence __buildOccurrence(String dependencyTypeKey, String depName, String symbol) { + final section = getYamlSection(pubspecStr, RegExp('^${dependencyTypeKey}:', multiLine: true)); + if (section == null) { + throw Exception('Unable to find section "$dependencyTypeKey" in pubspec.yaml'); + } + final sectionStart = pubspecStr.indexOf(section); + final depStart = section.indexOf(' ${depName}:') + 2; + final startOffset = sectionStart + depStart; + + return Occurrence( + symbol: symbol, + range: pubspecLineInfo.getRange(startOffset, depName.length) + ); + } + + final fileSymbol = [ + 'scip-dart', + 'pub', + pubspec.name, + pubspec.version!, + '`pubspec.yaml`/', + ].join(' '); + + final symbols = [ + SymbolInformation( + symbol: fileSymbol, + kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: [ + 'name: ${pubspec.name}', + 'version: ${pubspec.version}' + ].join('\n') + ) + ) + ]; + final occurrences = [ + Occurrence( + symbol: fileSymbol, + range: [0, 0, 0], + ) + ]; + + for (final dep in pubspec.dependencies.entries) { + final info = __buildSymbol(dep.key, dep.value); + symbols.add(info); + occurrences.add(__buildOccurrence('dependencies', dep.key, info.symbol)); + } + + for (final dep in pubspec.devDependencies.entries) { + final info = __buildSymbol(dep.key, dep.value); + symbols.add(info); + occurrences.add(__buildOccurrence('dev_dependencies', dep.key, info.symbol)); + } + + for (final dep in pubspec.dependencyOverrides.entries) { + final info = __buildSymbol(dep.key, dep.value); + symbols.add(info); + occurrences.add(__buildOccurrence('dependency_overrides', dep.key, info.symbol)); + } + + return Document( + language: Language.YAML.name, + relativePath: 'pubspec.yaml', + symbols: symbols, + occurrences: occurrences, + ); +} \ No newline at end of file diff --git a/lib/src/utils.dart b/lib/src/utils.dart index e4d19710..d12dca03 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:analyzer/source/line_info.dart'; import 'package:path/path.dart' as p; +import 'package:collection/collection.dart'; import 'package:scip_dart/src/flags.dart'; /// Returns a list of all the pubspec.yaml paths under a directory. @@ -41,3 +42,46 @@ extension LineInfoExtension on LineInfo { ]; } } + +String? getYamlSection( + String str, + Pattern startRegex, { + bool skipMatchedLine = false, +}) { + final indentSize = getYamlIndentSize(str); + + final match = startRegex.allMatches(str).firstOrNull; + if (match == null) return null; + + final startingCharacter = match.start; + + final sectionAndAfterLines = str.substring(startingCharacter).split('\n'); + final sectionKeyLine = sectionAndAfterLines[0]; + final sectionKeyIndentSize = + sectionKeyLine.length - sectionKeyLine.trimLeft().length; + + final inSectionIndent = + List.filled(sectionKeyIndentSize + indentSize, ' ').join(); + + final inSectionLines = sectionAndAfterLines + .skip(1) // skip the section key line + .takeWhile( + (line) => line.trim().isEmpty || line.startsWith(inSectionIndent)) + .toList(); + + return [if (!skipMatchedLine) sectionKeyLine, ...inSectionLines] + .where((line) => line.trim().isNotEmpty) + .join('\n'); +} + +int getYamlIndentSize(String str) { + final lines = str.split('\n'); + + for (var line in lines) { + if (line.startsWith(' ')) { + return line.length - line.trimLeft().length; + } + } + + return 0; +} diff --git a/pubspec.lock b/pubspec.lock index eefd4657..a9e96132 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -185,6 +185,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.4" + pubspec_lock_parse: + dependency: "direct main" + description: + name: pubspec_lock_parse + sha256: "020cb470287124c936c30ebfc2f927b287f275b7bf7fc2ab11577e592c017764" + url: "https://pub.dev" + source: hosted + version: "2.2.0" pubspec_parse: dependency: "direct main" description: @@ -193,6 +201,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.3" + pubspec_writer_extensions: + dependency: "direct main" + description: + name: pubspec_writer_extensions + sha256: b6a581f8198c3af726ea9d640009d97894262f2cf047589522626375bde21aa4 + url: "https://pub.dev" + source: hosted + version: "1.0.0" source_span: dependency: transitive description: @@ -257,5 +273,13 @@ packages: url: "https://pub.dev" source: hosted version: "3.1.2" + yaml_writer: + dependency: "direct main" + description: + name: yaml_writer + sha256: "76740047e3d4b1687c588bc8cb3466bea55a7d915a91bb23e99ea2a8c58337b6" + url: "https://pub.dev" + source: hosted + version: "1.0.3" sdks: dart: ">=2.19.0 <3.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index bf6ef4e4..930e0b1e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -16,6 +16,9 @@ dependencies: path: ^1.8.3 protobuf: ^3.1.0 pubspec_parse: ^1.2.1 + pubspec_lock_parse: ^2.2.0 + pubspec_writer_extensions: ^1.0.0 + yaml_writer: ^1.0.3 dev_dependencies: chalk: ^1.2.1 diff --git a/snapshots/input/staging-project/pubspec.lock b/snapshots/input/staging-project/pubspec.lock index 89b205e1..057ce5bf 100644 --- a/snapshots/input/staging-project/pubspec.lock +++ b/snapshots/input/staging-project/pubspec.lock @@ -1,5 +1,101 @@ # Generated by pub # See https://dart.dev/tools/pub/glossary#lockfile -packages: {} +packages: + args: + dependency: transitive + description: + name: args + sha256: eef6c46b622e0494a36c5a12d10d77fb4e855501a91c1b9ef9339326e58f0596 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + checked_yaml: + dependency: transitive + description: + name: checked_yaml + sha256: feb6bed21949061731a7a75fc5d2aa727cf160b91af9a3e464c5e3a32e28b5ff + url: "https://pub.dev" + source: hosted + version: "2.0.3" + collection: + dependency: transitive + description: + name: collection + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a + url: "https://pub.dev" + source: hosted + version: "1.18.0" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: b10a7b2ff83d83c777edba3c6a0f97045ddadd56c944e1a23a3fdf43a1bf4467 + url: "https://pub.dev" + source: hosted + version: "4.8.1" + meta: + dependency: transitive + description: + name: meta + sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + url: "https://pub.dev" + source: hosted + version: "1.16.0" + path: + dependency: transitive + description: + name: path + sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + url: "https://pub.dev" + source: hosted + version: "1.8.3" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + pubspec_lock_parse: + dependency: "direct main" + description: + name: pubspec_lock_parse + sha256: "020cb470287124c936c30ebfc2f927b287f275b7bf7fc2ab11577e592c017764" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + source_span: + dependency: transitive + description: + name: source_span + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" + url: "https://pub.dev" + source: hosted + version: "1.10.0" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: "75769501ea3489fca56601ff33454fe45507ea3bfb014161abc3b43ae25989d5" + url: "https://pub.dev" + source: hosted + version: "3.1.2" sdks: - dart: ">=2.18.0 <3.0.0" + dart: ">=2.19.0 <3.0.0" diff --git a/snapshots/input/staging-project/pubspec.yaml b/snapshots/input/staging-project/pubspec.yaml index a15a94b2..6a993d00 100644 --- a/snapshots/input/staging-project/pubspec.yaml +++ b/snapshots/input/staging-project/pubspec.yaml @@ -3,3 +3,7 @@ version: 1.0.0 environment: sdk: '>=2.19.0 <3.0.0' + + +dependencies: + pubspec_lock_parse: ^2.0.2 \ No newline at end of file diff --git a/snapshots/output/basic-project/pubspec.yaml b/snapshots/output/basic-project/pubspec.yaml new file mode 100755 index 00000000..ea4b27a5 --- /dev/null +++ b/snapshots/output/basic-project/pubspec.yaml @@ -0,0 +1,10 @@ + name: dart_test +// reference scip-dart pub dart_test 1.0.0 `pubspec.yaml`/ + version: 1.0.0 + + environment: + sdk: '>=2.19.0 <3.0.0' + + dependencies: + test: ^1.24.3 +// ^^^^ reference scip-dart pub test 1.24.3 `pubspec.yaml`/ diff --git a/snapshots/output/diagnostics/pubspec.yaml b/snapshots/output/diagnostics/pubspec.yaml new file mode 100755 index 00000000..91cf3231 --- /dev/null +++ b/snapshots/output/diagnostics/pubspec.yaml @@ -0,0 +1,10 @@ + name: dart_test_diagnostics +// reference scip-dart pub dart_test_diagnostics 1.0.0 `pubspec.yaml`/ + version: 1.0.0 + + environment: + sdk: '>=2.19.0 <3.0.0' + dependencies: + lints: ^2.0.1 +// ^^^^^ reference scip-dart pub lints 2.0.1 `pubspec.yaml`/ + diff --git a/snapshots/output/staging-project/pubspec.yaml b/snapshots/output/staging-project/pubspec.yaml new file mode 100755 index 00000000..8d4434a1 --- /dev/null +++ b/snapshots/output/staging-project/pubspec.yaml @@ -0,0 +1,11 @@ + name: dart_test +// reference scip-dart pub dart_test 1.0.0 `pubspec.yaml`/ + version: 1.0.0 + + environment: + sdk: '>=2.19.0 <3.0.0' + + + dependencies: + pubspec_lock_parse: ^2.0.2 +// ^^^^^^^^^^^^^^^^^^ reference scip-dart pub pubspec_lock_parse 2.2.0 `pubspec.yaml`/ From d58ccd7d4c15ac1494680f9434093061829c15e3 Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:51:04 -0600 Subject: [PATCH 2/6] opt into indexing pubspec --- bin/scip_dart.dart | 28 ++++++--- lib/src/indexer.dart | 102 ------------------------------- lib/src/pubspec_indexer.dart | 115 +++++++++++++++++++++++++++++++++++ pubspec.yaml | 2 +- 4 files changed, 137 insertions(+), 110 deletions(-) create mode 100644 lib/src/pubspec_indexer.dart diff --git a/bin/scip_dart.dart b/bin/scip_dart.dart index 3b0e2b4b..8d0f3831 100644 --- a/bin/scip_dart.dart +++ b/bin/scip_dart.dart @@ -7,6 +7,7 @@ import 'package:package_config/package_config.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; import 'package:path/path.dart' as p; import 'package:scip_dart/src/flags.dart'; +import 'package:scip_dart/src/pubspec_indexer.dart'; import 'package:scip_dart/src/version.dart'; Future main(List args) async { @@ -18,6 +19,11 @@ Future main(List args) async { help: 'The output file to write the index to. Use "-" to write to stdout', ) + ..addFlag( + 'index-pubspec', + defaultsTo: false, + help: 'Whether or not to index the pubspec.yaml file', + ) ..addFlag( 'performance', aliases: ['perf'], @@ -78,15 +84,23 @@ Future main(List args) async { final pubspecStr = pubspecFile.readAsStringSync(); final pubspec = Pubspec.parse(pubspecStr); - final pubspecLockFile = File(p.join(packageRoot, 'pubspec.lock')); - if (!pubspecLockFile.existsSync()) { - stderr.writeln('ERROR: Unable to locate pubspec.lock. Have you ran pub get?'); - exit(1); - } - final pubspecLock = PubspecLock.parse(pubspecLockFile.readAsStringSync()); + + final index = await indexPackage(packageRoot, packageConfig, pubspec); + if (result['index-pubspec'] as bool) { + final pubspecLockFile = File(p.join(packageRoot, 'pubspec.lock')); + if (!pubspecLockFile.existsSync()) { + stderr.writeln('ERROR: Unable to locate pubspec.lock. Have you ran pub get?'); + exit(1); + } + final pubspecLock = PubspecLock.parse(pubspecLockFile.readAsStringSync()); - final index = await indexPackage(packageRoot, packageConfig, pubspec, pubspecStr, pubspecLock); + index.documents.add(indexPubspec( + pubspec: pubspec, + pubspecStr: pubspecStr, + pubspecLock: pubspecLock, + )); + } if (result['output'] as String == '-') { stdout.add(index.writeToBuffer()); diff --git a/lib/src/indexer.dart b/lib/src/indexer.dart index e7648938..fdd69b0d 100644 --- a/lib/src/indexer.dart +++ b/lib/src/indexer.dart @@ -1,9 +1,7 @@ import 'package:analyzer/dart/analysis/analysis_context_collection.dart'; import 'package:analyzer/dart/analysis/results.dart'; -import 'package:analyzer/source/line_info.dart'; import 'package:path/path.dart' as p; import 'package:package_config/package_config.dart'; -import 'package:pubspec_lock_parse/pubspec_lock_parse.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; import 'package:scip_dart/src/flags.dart'; @@ -16,8 +14,6 @@ Future indexPackage( String root, PackageConfig packageConfig, Pubspec pubspec, - String pubspecStr, - PubspecLock pubspecLock, ) async { final dirPath = p.normalize(p.absolute(root)); @@ -94,107 +90,9 @@ Future indexPackage( print('Parsing Ast took: ${st.elapsedMilliseconds}ms'); } - documents.add(_indexPubspec(root, pubspec, pubspecStr, pubspecLock)); - return Index( metadata: metadata, documents: documents, externalSymbols: globalExternalSymbols, ); -} - - -Document _indexPubspec(String rootPath, Pubspec pubspec, String pubspecStr, PubspecLock lock) { - final pubspecLineInfo = LineInfo.fromContent(pubspecStr); - - SymbolInformation __buildSymbol(String depName, Dependency dep) { - final depVersion = lock.packages[depName]!.version.toString(); - - return SymbolInformation( - displayName: depName, - symbol: [ - 'scip-dart', - 'pub', - depName, - depVersion, - '`pubspec.yaml`/', - ].join(' '), - kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency - signatureDocumentation: Document( - language: Language.YAML.name, - text: [ - 'name: $depName', - 'version: $depVersion', - ].join('\n') - ) - ); - } - - Occurrence __buildOccurrence(String dependencyTypeKey, String depName, String symbol) { - final section = getYamlSection(pubspecStr, RegExp('^${dependencyTypeKey}:', multiLine: true)); - if (section == null) { - throw Exception('Unable to find section "$dependencyTypeKey" in pubspec.yaml'); - } - final sectionStart = pubspecStr.indexOf(section); - final depStart = section.indexOf(' ${depName}:') + 2; - final startOffset = sectionStart + depStart; - - return Occurrence( - symbol: symbol, - range: pubspecLineInfo.getRange(startOffset, depName.length) - ); - } - - final fileSymbol = [ - 'scip-dart', - 'pub', - pubspec.name, - pubspec.version!, - '`pubspec.yaml`/', - ].join(' '); - - final symbols = [ - SymbolInformation( - symbol: fileSymbol, - kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency - signatureDocumentation: Document( - language: Language.YAML.name, - text: [ - 'name: ${pubspec.name}', - 'version: ${pubspec.version}' - ].join('\n') - ) - ) - ]; - final occurrences = [ - Occurrence( - symbol: fileSymbol, - range: [0, 0, 0], - ) - ]; - - for (final dep in pubspec.dependencies.entries) { - final info = __buildSymbol(dep.key, dep.value); - symbols.add(info); - occurrences.add(__buildOccurrence('dependencies', dep.key, info.symbol)); - } - - for (final dep in pubspec.devDependencies.entries) { - final info = __buildSymbol(dep.key, dep.value); - symbols.add(info); - occurrences.add(__buildOccurrence('dev_dependencies', dep.key, info.symbol)); - } - - for (final dep in pubspec.dependencyOverrides.entries) { - final info = __buildSymbol(dep.key, dep.value); - symbols.add(info); - occurrences.add(__buildOccurrence('dependency_overrides', dep.key, info.symbol)); - } - - return Document( - language: Language.YAML.name, - relativePath: 'pubspec.yaml', - symbols: symbols, - occurrences: occurrences, - ); } \ No newline at end of file diff --git a/lib/src/pubspec_indexer.dart b/lib/src/pubspec_indexer.dart new file mode 100644 index 00000000..7cdbbc11 --- /dev/null +++ b/lib/src/pubspec_indexer.dart @@ -0,0 +1,115 @@ + +import 'package:analyzer/source/line_info.dart'; +import 'package:pubspec_lock_parse/pubspec_lock_parse.dart'; +import 'package:pubspec_parse/pubspec_parse.dart'; +import 'package:scip_dart/src/utils.dart'; + +import 'gen/scip.pb.dart'; + +Document indexPubspec({ + required Pubspec pubspec, + required String pubspecStr, + required PubspecLock pubspecLock, +}) { + final pubspecLineInfo = LineInfo.fromContent(pubspecStr); + + final fileSymbol = [ + 'scip-dart', + 'pub', + pubspec.name, + pubspec.version!, + '`pubspec.yaml`/', + ].join(' '); + + final symbols = [ + SymbolInformation( + symbol: fileSymbol, + kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: [ + 'name: ${pubspec.name}', + 'version: ${pubspec.version}' + ].join('\n') + ) + ) + ]; + final occurrences = [ + Occurrence( + symbol: fileSymbol, + range: [0, 0, 0], + ) + ]; + + final deps = { + DependencyKind.regular: pubspec.dependencies, + DependencyKind.dev: pubspec.devDependencies, + DependencyKind.override: pubspec.dependencyOverrides, + }; + for (final kind in deps.keys) { + for (final dep in deps[kind]!.entries) { + final info = _buildSymbol(dep.key, pubspecLock); + symbols.add(info); + occurrences.add(Occurrence( + symbol: info.symbol, + range: pubspecLineInfo.getDependencyRange(pubspecStr, dep.key, kind), + )); + } + } + + return Document( + language: Language.YAML.name, + relativePath: 'pubspec.yaml', + symbols: symbols, + occurrences: occurrences, + ); +} + +SymbolInformation _buildSymbol(String depName, PubspecLock lock) { + final depVersion = lock.packages[depName]?.version.toString(); + if (depVersion == null) { + throw Exception('Unable to find ${depName} in pubspec.lock. Have you ran pub get?'); + } + + return SymbolInformation( + displayName: depName, + symbol: [ + 'scip-dart', + 'pub', + depName, + depVersion, + '`pubspec.yaml`/', + ].join(' '), + kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: [ + 'name: $depName', + 'version: $depVersion', + ].join('\n') + ) + ); +} + +enum DependencyKind { + regular('^dependencies:'), + dev('^dev_dependencies:'), + override('^dependency_overrides:'); + + final String matcher; + const DependencyKind(this.matcher); +} + +extension _PubspecLineInfo on LineInfo { + List getDependencyRange(String pubspecStr, String name, DependencyKind kind) { + final section = getYamlSection(pubspecStr, RegExp(kind.matcher, multiLine: true)); + if (section == null) { + throw Exception('Unable to find section "${kind.matcher}" in pubspec.yaml'); + } + final sectionStart = pubspecStr.indexOf(section); + final depStart = section.indexOf(' ${name}:') + 2; + final startOffset = sectionStart + depStart; + + return getRange(startOffset, name.length); + } +} \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 930e0b1e..3381a747 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -4,7 +4,7 @@ description: generates scip bindings for dart files repository: https://github.com/Workiva/scip-dart environment: - sdk: ">=2.19.0 <3.0.0" + sdk: ">=2.19.6 <3.0.0" executables: scip_dart: From 2558fa4575fcf715d9666c4f16caa29e9455fde3 Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:52:37 -0600 Subject: [PATCH 3/6] added documentation --- lib/src/pubspec_indexer.dart | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/src/pubspec_indexer.dart b/lib/src/pubspec_indexer.dart index 7cdbbc11..d0c02c78 100644 --- a/lib/src/pubspec_indexer.dart +++ b/lib/src/pubspec_indexer.dart @@ -6,6 +6,18 @@ import 'package:scip_dart/src/utils.dart'; import 'gen/scip.pb.dart'; +/// Generates a scip document representation of a pubspec.yaml file. Output +/// specification looks like the following: +/// +/// ```yaml +/// # scip-dart pub foo_pkg 1.0.0 `pubspec.yaml`/ +/// name: foo_pkg +/// version: 1.0.0 +/// +/// dependencies: +/// some_pkg: ^1.0.5 +/// # ^^^^^^^^ scip-dart pub some_pkg 1.5.6 `pubspec.yaml`/ +/// ``` Document indexPubspec({ required Pubspec pubspec, required String pubspecStr, From 2cc48e4eb48bf52ead862a7e9b8d6b7b87f861d1 Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:55:03 -0600 Subject: [PATCH 4/6] fixed dep validator --- bin/scip_dart.dart | 8 ++-- lib/src/indexer.dart | 2 +- lib/src/pubspec_indexer.dart | 73 ++++++++++++++++++------------------ lib/src/utils.dart | 6 +-- pubspec.lock | 20 +--------- pubspec.yaml | 3 +- 6 files changed, 47 insertions(+), 65 deletions(-) diff --git a/bin/scip_dart.dart b/bin/scip_dart.dart index 8d0f3831..696f69bf 100644 --- a/bin/scip_dart.dart +++ b/bin/scip_dart.dart @@ -20,8 +20,8 @@ Future main(List args) async { 'The output file to write the index to. Use "-" to write to stdout', ) ..addFlag( - 'index-pubspec', - defaultsTo: false, + 'index-pubspec', + defaultsTo: false, help: 'Whether or not to index the pubspec.yaml file', ) ..addFlag( @@ -84,13 +84,13 @@ Future main(List args) async { final pubspecStr = pubspecFile.readAsStringSync(); final pubspec = Pubspec.parse(pubspecStr); - final index = await indexPackage(packageRoot, packageConfig, pubspec); if (result['index-pubspec'] as bool) { final pubspecLockFile = File(p.join(packageRoot, 'pubspec.lock')); if (!pubspecLockFile.existsSync()) { - stderr.writeln('ERROR: Unable to locate pubspec.lock. Have you ran pub get?'); + stderr.writeln( + 'ERROR: Unable to locate pubspec.lock. Have you ran pub get?'); exit(1); } final pubspecLock = PubspecLock.parse(pubspecLockFile.readAsStringSync()); diff --git a/lib/src/indexer.dart b/lib/src/indexer.dart index fdd69b0d..977f6829 100644 --- a/lib/src/indexer.dart +++ b/lib/src/indexer.dart @@ -95,4 +95,4 @@ Future indexPackage( documents: documents, externalSymbols: globalExternalSymbols, ); -} \ No newline at end of file +} diff --git a/lib/src/pubspec_indexer.dart b/lib/src/pubspec_indexer.dart index d0c02c78..493a0f3a 100644 --- a/lib/src/pubspec_indexer.dart +++ b/lib/src/pubspec_indexer.dart @@ -1,4 +1,3 @@ - import 'package:analyzer/source/line_info.dart'; import 'package:pubspec_lock_parse/pubspec_lock_parse.dart'; import 'package:pubspec_parse/pubspec_parse.dart'; @@ -8,19 +7,19 @@ import 'gen/scip.pb.dart'; /// Generates a scip document representation of a pubspec.yaml file. Output /// specification looks like the following: -/// +/// /// ```yaml /// # scip-dart pub foo_pkg 1.0.0 `pubspec.yaml`/ /// name: foo_pkg /// version: 1.0.0 -/// +/// /// dependencies: /// some_pkg: ^1.0.5 /// # ^^^^^^^^ scip-dart pub some_pkg 1.5.6 `pubspec.yaml`/ /// ``` Document indexPubspec({ - required Pubspec pubspec, - required String pubspecStr, + required Pubspec pubspec, + required String pubspecStr, required PubspecLock pubspecLock, }) { final pubspecLineInfo = LineInfo.fromContent(pubspecStr); @@ -35,16 +34,13 @@ Document indexPubspec({ final symbols = [ SymbolInformation( - symbol: fileSymbol, - kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency - signatureDocumentation: Document( - language: Language.YAML.name, - text: [ - 'name: ${pubspec.name}', - 'version: ${pubspec.version}' - ].join('\n') - ) - ) + symbol: fileSymbol, + kind: SymbolInformation_Kind + .UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: ['name: ${pubspec.name}', 'version: ${pubspec.version}'] + .join('\n'))) ]; final occurrences = [ Occurrence( @@ -80,27 +76,27 @@ Document indexPubspec({ SymbolInformation _buildSymbol(String depName, PubspecLock lock) { final depVersion = lock.packages[depName]?.version.toString(); if (depVersion == null) { - throw Exception('Unable to find ${depName} in pubspec.lock. Have you ran pub get?'); + throw Exception( + 'Unable to find ${depName} in pubspec.lock. Have you ran pub get?'); } return SymbolInformation( - displayName: depName, - symbol: [ - 'scip-dart', - 'pub', - depName, - depVersion, - '`pubspec.yaml`/', - ].join(' '), - kind: SymbolInformation_Kind.UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency - signatureDocumentation: Document( - language: Language.YAML.name, - text: [ - 'name: $depName', - 'version: $depVersion', - ].join('\n') - ) - ); + displayName: depName, + symbol: [ + 'scip-dart', + 'pub', + depName, + depVersion, + '`pubspec.yaml`/', + ].join(' '), + kind: SymbolInformation_Kind + .UnspecifiedKind, // TODO: Add SymbolInformation_Kind.Dependency and SymbolInformation_Kind.DevDependency + signatureDocumentation: Document( + language: Language.YAML.name, + text: [ + 'name: $depName', + 'version: $depVersion', + ].join('\n'))); } enum DependencyKind { @@ -113,10 +109,13 @@ enum DependencyKind { } extension _PubspecLineInfo on LineInfo { - List getDependencyRange(String pubspecStr, String name, DependencyKind kind) { - final section = getYamlSection(pubspecStr, RegExp(kind.matcher, multiLine: true)); + List getDependencyRange( + String pubspecStr, String name, DependencyKind kind) { + final section = + getYamlSection(pubspecStr, RegExp(kind.matcher, multiLine: true)); if (section == null) { - throw Exception('Unable to find section "${kind.matcher}" in pubspec.yaml'); + throw Exception( + 'Unable to find section "${kind.matcher}" in pubspec.yaml'); } final sectionStart = pubspecStr.indexOf(section); final depStart = section.indexOf(' ${name}:') + 2; @@ -124,4 +123,4 @@ extension _PubspecLineInfo on LineInfo { return getRange(startOffset, name.length); } -} \ No newline at end of file +} diff --git a/lib/src/utils.dart b/lib/src/utils.dart index d12dca03..dd0eb01a 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -44,7 +44,7 @@ extension LineInfoExtension on LineInfo { } String? getYamlSection( - String str, + String str, Pattern startRegex, { bool skipMatchedLine = false, }) { @@ -70,8 +70,8 @@ String? getYamlSection( .toList(); return [if (!skipMatchedLine) sectionKeyLine, ...inSectionLines] - .where((line) => line.trim().isNotEmpty) - .join('\n'); + .where((line) => line.trim().isNotEmpty) + .join('\n'); } int getYamlIndentSize(String str) { diff --git a/pubspec.lock b/pubspec.lock index a9e96132..389dc078 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -58,7 +58,7 @@ packages: source: hosted version: "2.0.3" collection: - dependency: transitive + dependency: "direct main" description: name: collection sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a @@ -201,14 +201,6 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.3" - pubspec_writer_extensions: - dependency: "direct main" - description: - name: pubspec_writer_extensions - sha256: b6a581f8198c3af726ea9d640009d97894262f2cf047589522626375bde21aa4 - url: "https://pub.dev" - source: hosted - version: "1.0.0" source_span: dependency: transitive description: @@ -273,13 +265,5 @@ packages: url: "https://pub.dev" source: hosted version: "3.1.2" - yaml_writer: - dependency: "direct main" - description: - name: yaml_writer - sha256: "76740047e3d4b1687c588bc8cb3466bea55a7d915a91bb23e99ea2a8c58337b6" - url: "https://pub.dev" - source: hosted - version: "1.0.3" sdks: - dart: ">=2.19.0 <3.0.0" + dart: ">=2.19.6 <3.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index 3381a747..0b8a5364 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -17,8 +17,7 @@ dependencies: protobuf: ^3.1.0 pubspec_parse: ^1.2.1 pubspec_lock_parse: ^2.2.0 - pubspec_writer_extensions: ^1.0.0 - yaml_writer: ^1.0.3 + collection: ^1.18.0 dev_dependencies: chalk: ^1.2.1 From 1e2a941d48ff94675f6387edf0ca721547e12494 Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:55:16 -0600 Subject: [PATCH 5/6] snaps --- snapshots/output/basic-project/pubspec.yaml | 10 ---------- snapshots/output/diagnostics/pubspec.yaml | 10 ---------- 2 files changed, 20 deletions(-) delete mode 100755 snapshots/output/basic-project/pubspec.yaml delete mode 100755 snapshots/output/diagnostics/pubspec.yaml diff --git a/snapshots/output/basic-project/pubspec.yaml b/snapshots/output/basic-project/pubspec.yaml deleted file mode 100755 index ea4b27a5..00000000 --- a/snapshots/output/basic-project/pubspec.yaml +++ /dev/null @@ -1,10 +0,0 @@ - name: dart_test -// reference scip-dart pub dart_test 1.0.0 `pubspec.yaml`/ - version: 1.0.0 - - environment: - sdk: '>=2.19.0 <3.0.0' - - dependencies: - test: ^1.24.3 -// ^^^^ reference scip-dart pub test 1.24.3 `pubspec.yaml`/ diff --git a/snapshots/output/diagnostics/pubspec.yaml b/snapshots/output/diagnostics/pubspec.yaml deleted file mode 100755 index 91cf3231..00000000 --- a/snapshots/output/diagnostics/pubspec.yaml +++ /dev/null @@ -1,10 +0,0 @@ - name: dart_test_diagnostics -// reference scip-dart pub dart_test_diagnostics 1.0.0 `pubspec.yaml`/ - version: 1.0.0 - - environment: - sdk: '>=2.19.0 <3.0.0' - dependencies: - lints: ^2.0.1 -// ^^^^^ reference scip-dart pub lints 2.0.1 `pubspec.yaml`/ - From ea2e20ebc5da1ece763e9ced7eb266d255724add Mon Sep 17 00:00:00 2001 From: Matthew Nitschke Date: Tue, 15 Apr 2025 10:56:21 -0600 Subject: [PATCH 6/6] index pubspecs in snapshot tests --- .github/workflows/ci.yaml | 4 ++-- Makefile | 4 ++-- snapshots/output/basic-project/pubspec.yaml | 10 ++++++++++ snapshots/output/diagnostics/pubspec.yaml | 10 ++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100755 snapshots/output/basic-project/pubspec.yaml create mode 100755 snapshots/output/diagnostics/pubspec.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 17bec578..d447a6c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,10 +44,10 @@ jobs: - name: Snapshots Diff Check run: | - dart run scip_dart ./snapshots/input/basic-project + dart run scip_dart ./snapshots/input/basic-project --index-pubspec ./scip snapshot --to ./snapshots/output/basic-project - dart run scip_dart ./snapshots/input/diagnostics + dart run scip_dart ./snapshots/input/diagnostics --index-pubspec ./scip snapshot --to ./snapshots/output/diagnostics if [[ -z "$(git status --porcelain ./snapshots/output)" ]]; diff --git a/Makefile b/Makefile index ae1a16ab..d05a5ee7 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ regen-snapshots: - dart run scip_dart ./snapshots/input/basic-project + dart run scip_dart ./snapshots/input/basic-project --index-pubspec scip snapshot --to ./snapshots/output/basic-project - dart run scip_dart ./snapshots/input/diagnostics + dart run scip_dart ./snapshots/input/diagnostics --index-pubspec scip snapshot --to ./snapshots/output/diagnostics run: diff --git a/snapshots/output/basic-project/pubspec.yaml b/snapshots/output/basic-project/pubspec.yaml new file mode 100755 index 00000000..ea4b27a5 --- /dev/null +++ b/snapshots/output/basic-project/pubspec.yaml @@ -0,0 +1,10 @@ + name: dart_test +// reference scip-dart pub dart_test 1.0.0 `pubspec.yaml`/ + version: 1.0.0 + + environment: + sdk: '>=2.19.0 <3.0.0' + + dependencies: + test: ^1.24.3 +// ^^^^ reference scip-dart pub test 1.24.3 `pubspec.yaml`/ diff --git a/snapshots/output/diagnostics/pubspec.yaml b/snapshots/output/diagnostics/pubspec.yaml new file mode 100755 index 00000000..91cf3231 --- /dev/null +++ b/snapshots/output/diagnostics/pubspec.yaml @@ -0,0 +1,10 @@ + name: dart_test_diagnostics +// reference scip-dart pub dart_test_diagnostics 1.0.0 `pubspec.yaml`/ + version: 1.0.0 + + environment: + sdk: '>=2.19.0 <3.0.0' + dependencies: + lints: ^2.0.1 +// ^^^^^ reference scip-dart pub lints 2.0.1 `pubspec.yaml`/ +