Parsing support for doc imports

Work towards #50702

* This adds a `docImport` field to the `Comment` class.
* Doc imports are parsed with the standard Fasta parser and
  AstBuilder.
* We add one static check that `deferred` isn't used. Other
  checks are needed.
* Many tests.
* We need to add support for line splits in a doc comment.
* I think doc comment imports need to be visted by AST visitors,
  but maybe I am wrong...

Change-Id: I06e2b6fe42ef5ce916d46d9a9db35334726677d0
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/322591
Commit-Queue: Samuel Rawlins <srawlins@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>

pkg/analysis_server/lib/src/services/correction/error_fix_status.yaml

@@ -3426,6 +3426,8 @@ WarningCode.DEPRECATED_MIXIN_FUNCTION:
     The fix is to remove `Function` from where it's referenced.
 WarningCode.DEPRECATED_NEW_IN_COMMENT_REFERENCE:
   status: hasFix
+WarningCode.DOC_IMPORT_CANNOT_BE_DEFERRED:
+  status: needsFix
 WarningCode.DUPLICATE_EXPORT:
   status: needsFix
   notes: |-

pkg/analyzer/lib/src/dart/ast/ast.dart

@@ -3223,6 +3223,9 @@ abstract final class Comment implements AstNode {
   @experimental
   List<MdCodeBlock> get codeBlocks;
 
+  @experimental
+  List<DocImport> get docImports;
+
   /// Return `true` if this is a block comment.
   bool get isBlock;
 
@@ -3271,6 +3274,9 @@ final class CommentImpl extends AstNodeImpl implements Comment {
   @override
   final List<MdCodeBlock> codeBlocks;
 
+  @override
+  final List<DocImport> docImports;
+
   /// Initialize a newly created comment. The list of [tokens] must contain at
   /// least one token. The [_type] is the type of the comment. The list of
   /// [references] can be empty if the comment does not contain any embedded
@@ -3280,6 +3286,7 @@ final class CommentImpl extends AstNodeImpl implements Comment {
     required CommentType type,
     required List<CommentReferenceImpl> references,
     required this.codeBlocks,
+    required this.docImports,
   }) : _type = type {
     _references._initialize(this, references);
   }
@@ -5307,6 +5314,21 @@ sealed class DirectiveImpl extends AnnotatedNodeImpl implements Directive {
   }
 }
 
+/// A documentation import, found in a doc comment.
+///
+/// Documentation imports are declared with `@docImport` at the start of a line
+/// of a documentation comment, followed by regular import elements (URI,
+/// optional prefix, optional combinators), ending with a semicolon.
+@experimental
+final class DocImport {
+  /// The offset of the starting text, '@docImport'.
+  int offset;
+
+  ImportDirective import;
+
+  DocImport({required this.offset, required this.import});
+}
+
 /// A do statement.
 ///
 ///    doStatement ::=

pkg/analyzer/lib/src/error/best_practices_verifier.dart

@@ -28,6 +28,7 @@ import 'package:analyzer/src/dart/resolver/scope.dart';
 import 'package:analyzer/src/error/annotation_verifier.dart';
 import 'package:analyzer/src/error/codes.dart';
 import 'package:analyzer/src/error/deprecated_member_use_verifier.dart';
+import 'package:analyzer/src/error/doc_import_verifier.dart';
 import 'package:analyzer/src/error/error_handler_verifier.dart';
 import 'package:analyzer/src/error/must_call_super_verifier.dart';
 import 'package:analyzer/src/error/null_safe_api_verifier.dart';
@@ -57,13 +58,13 @@ class BestPracticesVerifier extends RecursiveAstVisitor<void> {
   /// The type [Null].
   final InterfaceType _nullType;
 
-  /// The type system primitives
+  /// The type system primitives.
   final TypeSystemImpl _typeSystem;
 
   /// The inheritance manager to access interface type hierarchy.
   final InheritanceManager3 _inheritanceManager;
 
-  /// The current library
+  /// The current library.
   final LibraryElement _currentLibrary;
 
   final AnnotationVerifier _annotationVerifier;
@@ -78,6 +79,9 @@ class BestPracticesVerifier extends RecursiveAstVisitor<void> {
 
   final NullSafeApiVerifier _nullSafeApiVerifier;
 
+  late final DocImportVerifier _docImportVerifier =
+      DocImportVerifier(_errorReporter);
+
   /// The [WorkspacePackage] in which [_currentLibrary] is declared.
   final WorkspacePackage? _workspacePackage;
 
@@ -239,6 +243,14 @@ class BestPracticesVerifier extends RecursiveAstVisitor<void> {
     }
   }
 
+  @override
+  void visitComment(Comment node) {
+    for (var docImport in node.docImports) {
+      _docImportVerifier.docImport(docImport);
+    }
+    super.visitComment(node);
+  }
+
   @override
   void visitCommentReference(CommentReference node) {
     var newKeyword = node.newKeyword;

pkg/analyzer/lib/src/error/codes.g.dart

@@ -6046,6 +6046,11 @@ class WarningCode extends AnalyzerErrorCode {
     hasPublishedDocs: true,
   );
 
+  static const WarningCode DOC_IMPORT_CANNOT_BE_DEFERRED = WarningCode(
+    'DOC_IMPORT_CANNOT_BE_DEFERRED',
+    "Doc imports can't be deferred.",
+  );
+
   ///  Duplicate exports.
   ///
   ///  No parameters.

pkg/analyzer/lib/src/error/doc_import_verifier.dart

@@ -0,0 +1,25 @@
+// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/error/listener.dart';
+import 'package:analyzer/src/dart/ast/ast.dart';
+import 'package:analyzer/src/error/codes.g.dart';
+
+/// Verifies doc imports, written as `@docImport` inside doc comments.
+class DocImportVerifier {
+  final ErrorReporter _errorReporter;
+
+  DocImportVerifier(this._errorReporter);
+
+  void docImport(DocImport node) {
+    var deferredKeyword = node.import.deferredKeyword;
+    if (deferredKeyword != null) {
+      _errorReporter.reportErrorForToken(
+        WarningCode.DOC_IMPORT_CANNOT_BE_DEFERRED,
+        deferredKeyword,
+      );
+    }
+    // TODO(srawlins): Report warnings for configurations.
+  }
+}

pkg/analyzer/lib/src/error/error_code_values.g.dart

@@ -943,6 +943,7 @@ const List<ErrorCode> errorCodeValues = [
   WarningCode.DEPRECATED_IMPLEMENTS_FUNCTION,
   WarningCode.DEPRECATED_MIXIN_FUNCTION,
   WarningCode.DEPRECATED_NEW_IN_COMMENT_REFERENCE,
+  WarningCode.DOC_IMPORT_CANNOT_BE_DEFERRED,
   WarningCode.DUPLICATE_EXPORT,
   WarningCode.DUPLICATE_HIDDEN_NAME,
   WarningCode.DUPLICATE_IGNORE,

pkg/analyzer/lib/src/fasta/ast_builder.dart

@@ -5509,7 +5509,14 @@ class AstBuilder extends StackListener {
   @visibleForTesting
   CommentImpl parseDocComment(Token dartdoc) {
     // Build and return the comment.
-    return DocCommentBuilder(parser, dartdoc).build();
+    return DocCommentBuilder(
+      parser,
+      errorReporter.errorReporter,
+      uri,
+      _featureSet,
+      _lineInfo,
+      dartdoc,
+    ).build();
   }
 
   List<CollectionElementImpl> popCollectionElements(int count) {

pkg/analyzer/lib/src/fasta/doc_comment_builder.dart

@@ -9,8 +9,12 @@ import 'package:_fe_analyzer_shared/src/parser/util.dart'
 import 'package:_fe_analyzer_shared/src/scanner/scanner.dart';
 import 'package:_fe_analyzer_shared/src/scanner/token.dart' show StringToken;
 import 'package:_fe_analyzer_shared/src/scanner/token_constants.dart';
+import 'package:analyzer/dart/analysis/features.dart';
 import 'package:analyzer/dart/ast/token.dart' show Token, TokenType;
+import 'package:analyzer/error/listener.dart';
+import 'package:analyzer/source/line_info.dart';
 import 'package:analyzer/src/dart/ast/ast.dart';
+import 'package:analyzer/src/fasta/ast_builder.dart';
 
 /// Given that we have just found bracketed text within the given [comment],
 /// looks to see whether that text is (a) followed by a parenthesized link
@@ -75,13 +79,24 @@ int _findCommentReferenceEnd(String comment, int index, int end) {
 /// [Comment], which is ultimately built with [build].
 class DocCommentBuilder {
   final Parser parser;
+  final ErrorReporter? _errorReporter;
+  final Uri _uri;
+  final FeatureSet _featureSet;
+  final LineInfo _lineInfo;
   final List<CommentReferenceImpl> references = [];
   final List<MdCodeBlock> codeBlocks = [];
+  final List<DocImport> docImports = [];
   final Token startToken;
   final _CharacterSequence characterSequence;
 
-  DocCommentBuilder(this.parser, this.startToken)
-      : characterSequence = _CharacterSequence(startToken);
+  DocCommentBuilder(
+    this.parser,
+    this._errorReporter,
+    this._uri,
+    this._featureSet,
+    this._lineInfo,
+    this.startToken,
+  ) : characterSequence = _CharacterSequence(startToken);
 
   CommentImpl build() {
     parseDocComment();
@@ -101,6 +116,7 @@ class DocCommentBuilder {
       type: CommentType.DOCUMENTATION,
       references: references,
       codeBlocks: codeBlocks,
+      docImports: docImports,
     );
   }
 
@@ -126,7 +142,8 @@ class DocCommentBuilder {
       var fencedCodeBlockIndex = _fencedCodeBlockDelimiter(content);
       if (fencedCodeBlockIndex > -1) {
         _parseFencedCodeBlock(index: fencedCodeBlockIndex, content: content);
-      } else {
+      } else if (!_parseDocImport(
+          index: whitespaceEndIndex, content: content)) {
         _parseDocCommentLine(offset, content);
       }
       isPreviousLineEmpty = content.isEmpty;
@@ -201,6 +218,69 @@ class DocCommentBuilder {
     }
   }
 
+  /// Tries to parse a doc import at the beginning of a line of a doc comment,
+  /// returning whether this was successful.
+  ///
+  /// A doc import begins with `@docImport ` and then can contain any other
+  /// legal syntax that a regular Dart import can contain.
+  bool _parseDocImport({required int index, required String content}) {
+    const docImportLength = '@docImport '.length;
+    const importLength = 'import '.length;
+    if (!content.startsWith('@docImport ', index)) {
+      return false;
+    }
+
+    index = _readWhitespace(content, index + docImportLength);
+    var syntheticImport = 'import ${content.substring(index)}';
+
+    // TODO(srawlins): Handle multiple lines.
+    var sourceMap = [
+      (
+        offsetInDocImport: 0,
+        offsetInUnit: characterSequence._offset + (index - importLength),
+      )
+    ];
+
+    var scanner = DocImportStringScanner(
+      syntheticImport,
+      configuration: ScannerConfiguration(),
+      sourceMap: sourceMap,
+    );
+
+    var tokens = scanner.tokenize();
+    var result = ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors);
+    // Fasta pretends there is an additional line at EOF.
+    result.lineStarts.removeLast();
+    // For compatibility, there is already a first entry in lineStarts.
+    result.lineStarts.removeAt(0);
+
+    var token = result.tokens;
+    var docImportListener = AstBuilder(
+      _errorReporter,
+      _uri,
+      true /* isFullAst */,
+      _featureSet,
+      _lineInfo,
+    );
+    var parser = Parser(docImportListener);
+    docImportListener.parser = parser;
+    parser.parseUnit(token);
+
+    if (docImportListener.directives.isEmpty) {
+      return false;
+    }
+    var directive = docImportListener.directives.first;
+
+    if (directive is ImportDirectiveImpl) {
+      docImports.add(
+        DocImport(offset: characterSequence._offset, import: directive),
+      );
+      return true;
+    }
+
+    return false;
+  }
+
   /// Parses a fenced code block, starting with [content].
   ///
   /// The backticks of the opening delimiter start at [index].
@@ -464,10 +544,9 @@ class DocCommentBuilder {
 
   /// Reads past any opening whitespace in [content], returning the index after
   /// the last whitespace character.
-  int _readWhitespace(String content) {
-    if (content.isEmpty) return 0;
-    var index = 0;
+  int _readWhitespace(String content, [int index = 0]) {
     var length = content.length;
+    if (index >= length) return index;
     while (isWhitespace(content.codeUnitAt(index))) {
       index++;
       if (index >= length) {
@@ -478,6 +557,63 @@ class DocCommentBuilder {
   }
 }
 
+class DocImportStringScanner extends StringScanner {
+  /// A list of offset pairs; each contains an offset in [source], and the
+  /// associated offset in the source text from which [source] is derived.
+  ///
+  /// Always contains a mapping from 0 to the offset of the `@docImport` text.
+  ///
+  /// Additionally contains a mapping for the start of each new line.
+  ///
+  /// For example, given the unit text:
+  ///
+  /// ```dart
+  /// int x = 0;
+  /// /// Text.
+  /// /// @docImport 'dart:math'
+  /// // ignore: some_linter_rule
+  /// ///     as math
+  /// ///     show max;
+  /// int y = 0;
+  /// ```
+  ///
+  /// The source map for scanning the doc import will contain the following
+  /// pairs:
+  ///
+  /// * (0, 29) (29 is the offset of the `I` in `@docImport`.)
+  /// * (19, 80) (The offsets of the first character in the line with `as`.)
+  /// * (31, 96) (The offsets of the first character in the line with `show`.)
+  final List<({int offsetInDocImport, int offsetInUnit})> _sourceMap;
+
+  DocImportStringScanner(
+    super.source, {
+    super.configuration,
+    required List<({int offsetInDocImport, int offsetInUnit})> sourceMap,
+  }) : _sourceMap = sourceMap;
+
+  @override
+
+  /// The position of the start of the next token _in the unit_, not in
+  /// [source].
+  ///
+  /// This is used for constructing [Token] objects, for a Token's offset.
+  int get tokenStart => _toOffsetInUnit(super.tokenStart);
+
+  /// Maps [offset] to the corresponding offset in the unit.
+  int _toOffsetInUnit(int offset) {
+    for (var index = _sourceMap.length - 1; index > 0; index--) {
+      var (:offsetInDocImport, :offsetInUnit) = _sourceMap[index];
+      if (offset >= offsetInDocImport) {
+        var delta = offset - offsetInDocImport;
+        return offsetInUnit + delta;
+      }
+    }
+    var (:offsetInDocImport, :offsetInUnit) = _sourceMap[0];
+    var delta = offset - offsetInDocImport;
+    return offsetInUnit + delta;
+  }
+}
+
 /// An abstraction of the character sequences in either a single-line doc
 /// comment (which consists of a series of [Token]s) or a multi-line doc comment
 /// (which consists of a single [Token]).

pkg/analyzer/messages.yaml

@@ -21545,6 +21545,9 @@ StaticWarningCode:
       }
       ```
 WarningCode:
+  DOC_IMPORT_CANNOT_BE_DEFERRED:
+    problemMessage: "Doc imports can't be deferred."
+    hasPublishedDocs: false
   ARGUMENT_TYPE_NOT_ASSIGNABLE_TO_ERROR_HANDLER:
     problemMessage: "The argument type '{0}' can't be assigned to the parameter type '{1} Function(Object)' or '{1} Function(Object, StackTrace)'."
     hasPublishedDocs: true