diff --git a/pkg/analyzer/lib/src/dart/ast/ast.dart b/pkg/analyzer/lib/src/dart/ast/ast.dart index f4e596cead81..75ad143e53ae 100644 --- a/pkg/analyzer/lib/src/dart/ast/ast.dart +++ b/pkg/analyzer/lib/src/dart/ast/ast.dart @@ -3225,6 +3225,10 @@ sealed class CombinatorImpl extends AstNodeImpl implements Combinator { /// '/ **' (CHARACTER | [CommentReference])* '*/' /// | ('///' (CHARACTER - EOL)* EOL)+ abstract final class Comment implements AstNode { + /// The fenced code blocks parsed in this comment. + @experimental + List get fencedCodeBlocks; + /// Return `true` if this is a block comment. bool get isBlock; @@ -3270,6 +3274,9 @@ final class CommentImpl extends AstNodeImpl implements Comment { /// within it. final NodeListImpl _references = NodeListImpl._(); + @override + final List fencedCodeBlocks; + /// Initialize a newly created comment. The list of [tokens] must contain at /// least one token. The [_type] is the type of the comment. The list of /// [references] can be empty if the comment does not contain any embedded @@ -3278,6 +3285,7 @@ final class CommentImpl extends AstNodeImpl implements Comment { required this.tokens, required CommentType type, required List references, + required this.fencedCodeBlocks, }) : _type = type { _references._initialize(this, references); } @@ -11942,6 +11950,45 @@ final class MapPatternImpl extends DartPatternImpl implements MapPattern { } } +/// A Markdown fenced code block found in a documentation comment. +@experimental +final class MdFencedCodeBlock { + /// The 'info string'. + /// + /// This includes any text following the opening backticks. For example, in + /// a fenced code block starting with "```dart", the info string is "dart". + /// + /// If no text follows the opening backticks, the info string is `null`. + /// + /// See CommonMark specification at + /// . + final String? infoString; + + /// Information about the comment lines that make up this code block. + final List lines; + + MdFencedCodeBlock({ + required this.infoString, + required List lines, + }) : lines = List.of(lines, growable: false); +} + +/// A Markdown fenced code block line found in a documentation comment. +@experimental +final class MdFencedCodeBlockLine { + /// The offset of the start of the fenced code block, from the beginning of + /// compilation unit. + final int offset; + + /// The length of the fenced code block. + final int length; + + MdFencedCodeBlockLine({ + required this.offset, + required this.length, + }); +} + /// A method declaration. /// /// methodDeclaration ::= diff --git a/pkg/analyzer/lib/src/fasta/ast_builder.dart b/pkg/analyzer/lib/src/fasta/ast_builder.dart index 910d7c4d563a..a6cb4ad56c40 100644 --- a/pkg/analyzer/lib/src/fasta/ast_builder.dart +++ b/pkg/analyzer/lib/src/fasta/ast_builder.dart @@ -47,8 +47,6 @@ import 'package:_fe_analyzer_shared/src/parser/parser.dart' import 'package:_fe_analyzer_shared/src/parser/quote.dart'; import 'package:_fe_analyzer_shared/src/parser/stack_listener.dart' show NullValues, StackListener; -import 'package:_fe_analyzer_shared/src/parser/util.dart' - show isLetter, isLetterOrDigit, isWhitespace, optional; import 'package:_fe_analyzer_shared/src/scanner/errors.dart' show translateErrorToken; import 'package:_fe_analyzer_shared/src/scanner/scanner.dart'; @@ -65,10 +63,12 @@ import 'package:analyzer/src/dart/analysis/experiments.dart'; import 'package:analyzer/src/dart/ast/ast.dart'; import 'package:analyzer/src/dart/ast/extensions.dart'; import 'package:analyzer/src/dart/error/syntactic_errors.dart'; +import 'package:analyzer/src/fasta/doc_comment_builder.dart'; import 'package:analyzer/src/fasta/error_converter.dart'; import 'package:analyzer/src/generated/utilities_dart.dart'; import 'package:analyzer/src/summary2/ast_binary_tokens.dart'; import 'package:collection/collection.dart'; +import 'package:meta/meta.dart'; import 'package:pub_semver/pub_semver.dart'; /// A parser listener that builds the analyzer's AST structure. @@ -5505,65 +5505,18 @@ class AstBuilder extends StackListener { throw UnsupportedError(message.problemMessage); } - /// Given that we have just found bracketed text within the given [comment], - /// look to see whether that text is (a) followed by a parenthesized link - /// address, (b) followed by a colon, or (c) followed by optional whitespace - /// and another square bracket. - /// - /// [rightIndex] is the index of the right bracket. Return `true` if the - /// bracketed text is followed by a link address. - /// - /// This method uses the syntax described by the - /// markdown - /// project. - bool isLinkText(String comment, int rightIndex) { - var length = comment.length; - var index = rightIndex + 1; - if (index >= length) { - return false; - } - var ch = comment.codeUnitAt(index); - if (ch == 0x28 || ch == 0x3A) { - return true; - } - while (isWhitespace(ch)) { - index = index + 1; - if (index >= length) { - return false; - } - ch = comment.codeUnitAt(index); - } - return ch == 0x5B; - } - /// Return `true` if [token] is either `null` or is the symbol or keyword /// [value]. bool optionalOrNull(String value, Token? token) { return token == null || identical(value, token.stringValue); } - /// Parse the comment references in a sequence of comment tokens where + /// Parses the comment references in a sequence of comment tokens where /// [dartdoc] is the first token in the sequence. - List parseCommentReferences(Token dartdoc) { - // Parse dartdoc into potential comment reference source/offset pairs. - var sourcesAndOffsets = dartdoc.lexeme.startsWith('///') - ? _parseReferencesInSingleLineComments(dartdoc) - : _parseReferencesInMultiLineComment(dartdoc); - - var references = []; - // Parse each of the source/offset pairs into actual comment references. - for (var (:source, :offset) in sourcesAndOffsets) { - var result = scanString(source); - if (!result.hasErrors) { - var token = result.tokens; - var reference = _parseOneCommentReference(token, offset); - if (reference != null) { - references.add(reference); - } - } - } - - return references; + @visibleForTesting + CommentImpl parseDocComment(Token dartdoc) { + // Build and return the comment. + return DocCommentBuilder(parser, dartdoc).build(); } List popCollectionElements(int count) { @@ -5791,51 +5744,7 @@ class AstBuilder extends StackListener { } } - // Build and return the comment. - var references = parseCommentReferences(dartdoc); - List tokens = [dartdoc]; - if (dartdoc.lexeme.startsWith('///')) { - dartdoc = dartdoc.next; - while (dartdoc != null) { - if (dartdoc.lexeme.startsWith('///')) { - tokens.add(dartdoc); - } - dartdoc = dartdoc.next; - } - } - return CommentImpl( - tokens: tokens, - type: CommentType.DOCUMENTATION, - references: references, - ); - } - - /// Given a comment reference without a closing `]`, search for a possible - /// place where `]` should be. - int _findCommentReferenceEnd(String comment, int index, int end) { - // Find the end of the identifier if there is one. - if (index >= end || !isLetter(comment.codeUnitAt(index))) { - return index; - } - while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) { - ++index; - } - - // Check for a trailing `.`. - if (index >= end || comment.codeUnitAt(index) != 0x2E /* `.` */) { - return index; - } - ++index; - - // Find end of the identifier after the `.`. - if (index >= end || !isLetter(comment.codeUnitAt(index))) { - return index; - } - ++index; - while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) { - ++index; - } - return index; + return parseDocComment(dartdoc); } void _handleInstanceCreation(Token? token) { @@ -5859,298 +5768,6 @@ class AstBuilder extends StackListener { ); } - /// Parses the comment references in the text between [start] inclusive - /// and [end] exclusive. - /// - /// Returns information about the comment references as a list of records, - /// each with a `source` field and an `offset` field. The `source` is the text - /// between the delimiting `[` and `]` characters, not including them. The - /// `offset` is the offset of the comment reference in the containing - /// compilation unit. - /// - /// For example, for the text `/// [a] and [b.c].`, two records are returned: - /// `(source: 'a', offset: 5)` and `(source: 'b.c', offset: 13)` (assuming the - /// comment is the beginning of the compilation unit). - List<({String source, int offset})> _parseCommentReferencesInText( - Token commentToken, int start, int end) { - var comment = commentToken.lexeme; - var references = <({String source, int offset})>[]; - var index = start; - while (index < end) { - var ch = comment.codeUnitAt(index); - if (ch == 0x5B /* `[` */) { - ++index; - if (index < end && comment.codeUnitAt(index) == 0x3A /* `:` */) { - // Skip old-style code block. - index = comment.indexOf(':]', index + 1) + 1; - if (index == 0 || index > end) { - break; - } - } else { - var referenceStart = index; - index = comment.indexOf(']', index); - if (index == -1 || index >= end) { - // Recovery: terminating ']' is not typed yet. - index = _findCommentReferenceEnd(comment, referenceStart, end); - } - if (ch != 0x27 /* `'` */ && ch != 0x22 /* `"` */) { - if (isLinkText(comment, index)) { - // TODO(brianwilkerson) Handle the case where there's a library - // URI in the link text. - } else { - references.add(( - source: comment.substring(referenceStart, index), - offset: commentToken.charOffset + referenceStart, - )); - } - } - } - } else if (ch == 0x60 /* '`' */) { - // Skip inline code block if there is both starting '`' and ending '`'. - var endCodeBlock = comment.indexOf('`', index + 1); - if (endCodeBlock != -1 && endCodeBlock < end) { - index = endCodeBlock; - } - } - ++index; - } - return references; - } - - /// Parses the text in a single comment reference. - /// - /// Returns `null` if the text could not be parsed as a comment reference. - CommentReferenceImpl? _parseOneCommentReference( - Token token, int referenceOffset) { - var begin = token; - Token? newKeyword; - if (optional('new', token)) { - newKeyword = token; - token = token.next!; - } - Token? firstToken, firstPeriod, secondToken, secondPeriod; - if (token.isIdentifier && optional('.', token.next!)) { - secondToken = token; - secondPeriod = token.next!; - if (secondPeriod.next!.isIdentifier && - optional('.', secondPeriod.next!.next!)) { - firstToken = secondToken; - firstPeriod = secondPeriod; - secondToken = secondPeriod.next!; - secondPeriod = secondToken.next!; - } - var identifier = secondPeriod.next!; - if (identifier.kind == KEYWORD_TOKEN && optional('new', identifier)) { - // Treat `new` after `.` is as an identifier so that it can represent an - // unnamed constructor. This support is separate from the - // constructor-tearoffs feature. - parser.rewriter.replaceTokenFollowing( - secondPeriod, - StringToken(TokenType.IDENTIFIER, identifier.lexeme, - identifier.charOffset)); - } - token = secondPeriod.next!; - } - if (token.isEof) { - // Recovery: Insert a synthetic identifier for code completion - token = parser.rewriter.insertSyntheticIdentifier( - secondPeriod ?? newKeyword ?? parser.syntheticPreviousToken(token)); - if (begin == token.next!) { - begin = token; - } - } - Token? operatorKeyword; - if (optional('operator', token)) { - operatorKeyword = token; - token = token.next!; - } - if (token.isUserDefinableOperator) { - if (token.next!.isEof) { - return _parseOneCommentReferenceRest( - begin, - referenceOffset, - newKeyword, - firstToken, - firstPeriod, - secondToken, - secondPeriod, - token, - ); - } - } else { - token = operatorKeyword ?? token; - if (token.next!.isEof) { - if (token.isIdentifier) { - return _parseOneCommentReferenceRest( - begin, - referenceOffset, - newKeyword, - firstToken, - firstPeriod, - secondToken, - secondPeriod, - token, - ); - } - var keyword = token.keyword; - if (newKeyword == null && - secondToken == null && - (keyword == Keyword.THIS || - keyword == Keyword.NULL || - keyword == Keyword.TRUE || - keyword == Keyword.FALSE)) { - // TODO(brianwilkerson) If we want to support this we will need to - // extend the definition of CommentReference to take an expression - // rather than an identifier. For now we just ignore it to reduce the - // number of errors produced, but that's probably not a valid long - // term approach. - } - } - } - return null; - } - - /// Parses the parameters into a [CommentReferenceImpl]. - /// - /// If the reference begins with `new `, then pass the Token associated with - /// that text as [newToken]. - /// - /// If the reference contains a single identifier or operator (aside from the - /// optional [newToken]), then pass the associated Token as - /// [identifierOrOperator]. - /// - /// If the reference contains two identifiers separated by a period, then pass - /// the associated Tokens as [secondToken], [secondPeriod], and - /// [identifierOrOperator], in lexical order. - // TODO(srawlins): Rename the parameters or refactor this code to avoid the - // confusion of `null` values for the "first*" parameters and non-`null` values - // for the "second*" parameters. - /// - /// If the reference contains three identifiers, each separated by a period, - /// then pass the associated Tokens as [firstToken], [firstPeriod], - /// [secondToken], [secondPeriod], and [identifierOrOperator]. - CommentReferenceImpl _parseOneCommentReferenceRest( - Token begin, - int referenceOffset, - Token? newKeyword, - Token? firstToken, - Token? firstPeriod, - Token? secondToken, - Token? secondPeriod, - Token identifierOrOperator) { - // Adjust the token offsets to match the enclosing comment token. - var token = begin; - do { - token.offset += referenceOffset; - token = token.next!; - } while (!token.isEof); - - var identifier = SimpleIdentifierImpl(identifierOrOperator); - if (firstToken != null) { - var target = PrefixedIdentifierImpl( - prefix: SimpleIdentifierImpl(firstToken), - period: firstPeriod!, - identifier: SimpleIdentifierImpl(secondToken!), - ); - var expression = PropertyAccessImpl( - target: target, - operator: secondPeriod!, - propertyName: identifier, - ); - return CommentReferenceImpl( - newKeyword: newKeyword, - expression: expression, - ); - } else if (secondToken != null) { - var expression = PrefixedIdentifierImpl( - prefix: SimpleIdentifierImpl(secondToken), - period: secondPeriod!, - identifier: identifier, - ); - return CommentReferenceImpl( - newKeyword: newKeyword, - expression: expression, - ); - } else { - return CommentReferenceImpl( - newKeyword: newKeyword, - expression: identifier, - ); - } - } - - /// Parses the comment references in a multi-line comment token. - List<({String source, int offset})> _parseReferencesInMultiLineComment( - Token multiLineDoc) { - var comment = multiLineDoc.lexeme; - assert(comment.startsWith('/**')); - var references = <({String source, int offset})>[]; - var length = comment.length; - var start = 3; - var inCodeBlock = false; - var codeBlock = comment.indexOf('```', /* start = */ 3); - if (codeBlock == -1) { - codeBlock = length; - } - while (start < length) { - if (isWhitespace(comment.codeUnitAt(start))) { - ++start; - continue; - } - var end = comment.indexOf('\n', start); - if (end == -1) { - end = length; - } - if (codeBlock < end) { - inCodeBlock = !inCodeBlock; - codeBlock = comment.indexOf('```', end); - if (codeBlock == -1) { - codeBlock = length; - } - } - if (!inCodeBlock && !comment.startsWith('* ', start)) { - references - .addAll(_parseCommentReferencesInText(multiLineDoc, start, end)); - } - start = end + 1; - } - return references; - } - - /// Parse the comment references in a sequence of single line comment tokens - /// where [token] is the first comment token in the sequence. - /// Return the number of comment references parsed. - List<({String source, int offset})> _parseReferencesInSingleLineComments( - Token? token) { - var references = <({String source, int offset})>[]; - var inCodeBlock = false; - while (token != null && !token.isEof) { - var comment = token.lexeme; - if (comment.startsWith('///')) { - if (comment.indexOf('```', /* start = */ 3) != -1) { - inCodeBlock = !inCodeBlock; - } - if (!inCodeBlock) { - bool parseReferences; - if (comment.startsWith('/// ')) { - var previousComment = token.previous?.lexeme; - parseReferences = previousComment != null && - previousComment.startsWith('///') && - previousComment.trim().length > 3; - } else { - parseReferences = true; - } - if (parseReferences) { - references.addAll(_parseCommentReferencesInText( - token, /* start = */ 3, comment.length)); - } - } - } - token = token.next; - } - return references; - } - List _popNamedTypeList({ required ErrorCode errorCode, }) { diff --git a/pkg/analyzer/lib/src/fasta/doc_comment_builder.dart b/pkg/analyzer/lib/src/fasta/doc_comment_builder.dart new file mode 100644 index 000000000000..3360f8f6d073 --- /dev/null +++ b/pkg/analyzer/lib/src/fasta/doc_comment_builder.dart @@ -0,0 +1,472 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:_fe_analyzer_shared/src/parser/parser.dart' + show optional, Parser; +import 'package:_fe_analyzer_shared/src/parser/util.dart' + show isLetter, isLetterOrDigit, isWhitespace, optional; +import 'package:_fe_analyzer_shared/src/scanner/scanner.dart'; +import 'package:_fe_analyzer_shared/src/scanner/token.dart' + show DocumentationCommentToken, StringToken; +import 'package:_fe_analyzer_shared/src/scanner/token_constants.dart'; +import 'package:analyzer/dart/ast/token.dart' show Token, TokenType; +import 'package:analyzer/src/dart/ast/ast.dart'; + +/// Given that we have just found bracketed text within the given [comment], +/// looks to see whether that text is (a) followed by a parenthesized link +/// address, (b) followed by a colon, or (c) followed by optional whitespace +/// and another square bracket. +/// +/// [rightIndex] is the index of the right bracket. Return `true` if the +/// bracketed text is followed by a link address. +/// +/// This method uses the syntax described by the +/// markdown +/// project. +bool isLinkText(String comment, int rightIndex) { + var length = comment.length; + var index = rightIndex + 1; + if (index >= length) { + return false; + } + var ch = comment.codeUnitAt(index); + if (ch == 0x28 || ch == 0x3A) { + return true; + } + while (isWhitespace(ch)) { + index = index + 1; + if (index >= length) { + return false; + } + ch = comment.codeUnitAt(index); + } + return ch == 0x5B; +} + +/// Given a comment reference without a closing `]`, search for a possible +/// place where `]` should be. +int _findCommentReferenceEnd(String comment, int index, int end) { + // Find the end of the identifier if there is one. + if (index >= end || !isLetter(comment.codeUnitAt(index))) { + return index; + } + while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) { + ++index; + } + + // Check for a trailing `.`. + if (index >= end || comment.codeUnitAt(index) != 0x2E /* `.` */) { + return index; + } + ++index; + + // Find end of the identifier after the `.`. + if (index >= end || !isLetter(comment.codeUnitAt(index))) { + return index; + } + ++index; + while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) { + ++index; + } + return index; +} + +/// A class which temporarily stores data for a [CommentType.DOCUMENTATION]-type +/// [Comment], which is ultimately built with [build]. +class DocCommentBuilder { + final Parser parser; + final List references = []; + final List fencedCodeBlocks = []; + final Token startToken; + + DocCommentBuilder(this.parser, this.startToken); + + CommentImpl build() { + parseDocComment(); + var tokens = [startToken]; + Token? token = startToken; + if (token.lexeme.startsWith('///')) { + token = token.next; + while (token != null) { + if (token.lexeme.startsWith('///')) { + tokens.add(token); + } + token = token.next; + } + } + return CommentImpl( + tokens: tokens, + type: CommentType.DOCUMENTATION, + references: references, + fencedCodeBlocks: fencedCodeBlocks, + ); + } + + /// Parses a documentation comment. + /// + /// All parsed data is added to the fields on this builder. + void parseDocComment() { + // TODO(srawlins): This could be refactored into something more like a + // proper state machine. + var fromSingleLine = false; + var token = startToken; + if (token.lexeme.startsWith('///')) { + token = _joinSingleLineDocCommentTokens(token); + fromSingleLine = true; + } + var comment = token.lexeme; + if (!fromSingleLine) { + assert(comment.startsWith('/**')); + } + var length = comment.length; + // The offset, from the beginning of [comment], of the start of each line. + var start = 0; + // The offset, from the beginning of [comment], of the start of the content + // of each line. + var contentStart = 0; + int? fencedCodeBlockOffset; + String? fencedCodeBlockInfoString; + var isPreviousLineEmpty = true; + var fencedCodeBlockLines = []; + var possibleFencedCodeBlockIndex = comment.indexOf('```'); + if (possibleFencedCodeBlockIndex == -1) { + // This indicates that there is no fenced code block before the end of the + // comment. + possibleFencedCodeBlockIndex = length; + } + while (start < length) { + if (isWhitespace(comment.codeUnitAt(start))) { + ++start; + continue; + } + var end = comment.indexOf('\n', start); + if (end == -1) { + end = length; + } + if (fromSingleLine && !comment.startsWith('///', start)) { + // This must be a non-doc comment line, like a blank line or a `//` + // comment. + start = end + 1; + continue; + } + if (fromSingleLine) { + contentStart = start + 3; + } else { + contentStart = + comment.startsWith('* ', start) ? start + '* '.length : start; + } + if (fencedCodeBlockOffset != null) { + fencedCodeBlockLines.add( + MdFencedCodeBlockLine( + offset: contentStart, + length: end - contentStart, + ), + ); + } + if (possibleFencedCodeBlockIndex < end) { + if (fencedCodeBlockOffset == null) { + // This is the start of a fenced code block. + fencedCodeBlockOffset = + token.charOffset + possibleFencedCodeBlockIndex; + fencedCodeBlockInfoString = comment + .substring(possibleFencedCodeBlockIndex + '```'.length, end) + .trim(); + if (fencedCodeBlockInfoString.isEmpty) { + fencedCodeBlockInfoString = null; + } + fencedCodeBlockLines.add( + MdFencedCodeBlockLine( + offset: contentStart, + length: end - contentStart, + ), + ); + } else { + // This ends a fenced code block. + fencedCodeBlocks.add( + MdFencedCodeBlock( + infoString: fencedCodeBlockInfoString, + lines: fencedCodeBlockLines, + ), + ); + fencedCodeBlockOffset = null; + fencedCodeBlockInfoString = null; + fencedCodeBlockLines.clear(); + } + // Set the index of the next fenced code block delimiters. + possibleFencedCodeBlockIndex = comment.indexOf('```', end); + if (possibleFencedCodeBlockIndex == -1) { + possibleFencedCodeBlockIndex = length; + } + } + if (fencedCodeBlockOffset == null) { + var isIndentedCodeBlock = fromSingleLine + ? isPreviousLineEmpty && comment.startsWith('/// ', start) + : comment.startsWith('* ', start); + if (!isIndentedCodeBlock) { + _parseDocCommentLine(token, start, end); + } + } + // Mark the previous line as being empty if this function is called with + // a comment Token derived from a single-line comment Token, and the + // line is 3 characters long, which is exactly + isPreviousLineEmpty = + fromSingleLine && comment.substring(start, end) == '///'; + start = end + 1; + } + + // Recover a non-terminating code block. + if (fencedCodeBlockOffset != null) { + fencedCodeBlocks.add( + MdFencedCodeBlock( + infoString: fencedCodeBlockInfoString, + lines: fencedCodeBlockLines, + ), + ); + } + } + + /// Joins [startToken] with all of its following tokens. + /// + /// This should only be used to parse the contents of the doc comment text. + Token _joinSingleLineDocCommentTokens( + Token startToken, + ) { + var offset = startToken.offset; + var buffer = StringBuffer(); + buffer.writeln(startToken.lexeme); + var end = startToken.end; + var token = startToken.next; + while (token != null && !token.isEof) { + var gap = token.offset - (end + 1); + buffer.write('\n' * gap); + buffer.writeln(token.lexeme); + end = token.end; + token = token.next; + } + return DocumentationCommentToken( + TokenType.SINGLE_LINE_COMMENT, + buffer.toString(), + offset, + ); + } + + /// Parses the comment references in the text between [start] inclusive + /// and [end] exclusive. + void _parseDocCommentLine( + Token commentToken, + int start, + int end, + ) { + var comment = commentToken.lexeme; + var index = start; + while (index < end) { + var ch = comment.codeUnitAt(index); + if (ch == 0x5B /* `[` */) { + ++index; + if (index < end && comment.codeUnitAt(index) == 0x3A /* `:` */) { + // Skip old-style code block. + index = comment.indexOf(':]', index + 1) + 1; + if (index == 0 || index > end) { + break; + } + } else { + var referenceStart = index; + index = comment.indexOf(']', index); + if (index == -1 || index >= end) { + // Recovery: terminating ']' is not typed yet. + index = _findCommentReferenceEnd(comment, referenceStart, end); + } + if (ch != 0x27 /* `'` */ && ch != 0x22 /* `"` */) { + if (isLinkText(comment, index)) { + // TODO(brianwilkerson) Handle the case where there's a library + // URI in the link text. + } else { + var reference = _parseOneCommentReference( + comment.substring(referenceStart, index), + commentToken.charOffset + referenceStart, + ); + if (reference != null) { + references.add(reference); + } + } + } + } + } else if (ch == 0x60 /* '`' */) { + // Skip inline code block if there is both starting '`' and ending '`'. + var endCodeBlock = comment.indexOf('`', index + 1); + if (endCodeBlock != -1 && endCodeBlock < end) { + index = endCodeBlock; + } + } + ++index; + } + } + + /// Parses the [source] text, found at [offset] in a single comment reference. + /// + /// Returns `null` if the text could not be parsed as a comment reference. + CommentReferenceImpl? _parseOneCommentReference(String source, int offset) { + var result = scanString(source); + if (result.hasErrors) { + return null; + } + var token = result.tokens; + var begin = token; + Token? newKeyword; + if (optional('new', token)) { + newKeyword = token; + token = token.next!; + } + Token? firstToken, firstPeriod, secondToken, secondPeriod; + if (token.isIdentifier && optional('.', token.next!)) { + secondToken = token; + secondPeriod = token.next!; + if (secondPeriod.next!.isIdentifier && + optional('.', secondPeriod.next!.next!)) { + firstToken = secondToken; + firstPeriod = secondPeriod; + secondToken = secondPeriod.next!; + secondPeriod = secondToken.next!; + } + var identifier = secondPeriod.next!; + if (identifier.kind == KEYWORD_TOKEN && optional('new', identifier)) { + // Treat `new` after `.` is as an identifier so that it can represent an + // unnamed constructor. This support is separate from the + // constructor-tearoffs feature. + parser.rewriter.replaceTokenFollowing( + secondPeriod, + StringToken(TokenType.IDENTIFIER, identifier.lexeme, + identifier.charOffset)); + } + token = secondPeriod.next!; + } + if (token.isEof) { + // Recovery: Insert a synthetic identifier for code completion + token = parser.rewriter.insertSyntheticIdentifier( + secondPeriod ?? newKeyword ?? parser.syntheticPreviousToken(token)); + if (begin == token.next!) { + begin = token; + } + } + Token? operatorKeyword; + if (optional('operator', token)) { + operatorKeyword = token; + token = token.next!; + } + if (token.isUserDefinableOperator) { + if (token.next!.isEof) { + return _parseOneCommentReferenceRest( + begin, + offset, + newKeyword, + firstToken, + firstPeriod, + secondToken, + secondPeriod, + token, + ); + } + } else { + token = operatorKeyword ?? token; + if (token.next!.isEof) { + if (token.isIdentifier) { + return _parseOneCommentReferenceRest( + begin, + offset, + newKeyword, + firstToken, + firstPeriod, + secondToken, + secondPeriod, + token, + ); + } + var keyword = token.keyword; + if (newKeyword == null && + secondToken == null && + (keyword == Keyword.THIS || + keyword == Keyword.NULL || + keyword == Keyword.TRUE || + keyword == Keyword.FALSE)) { + // TODO(brianwilkerson) If we want to support this we will need to + // extend the definition of CommentReference to take an expression + // rather than an identifier. For now we just ignore it to reduce the + // number of errors produced, but that's probably not a valid long + // term approach. + } + } + } + return null; + } + + /// Parses the parameters into a [CommentReferenceImpl]. + /// + /// If the reference begins with `new `, then pass the Token associated with + /// that text as [newToken]. + /// + /// If the reference contains a single identifier or operator (aside from the + /// optional [newToken]), then pass the associated Token as + /// [identifierOrOperator]. + /// + /// If the reference contains two identifiers separated by a period, then pass + /// the associated Tokens as [secondToken], [secondPeriod], and + /// [identifierOrOperator], in lexical order. + // TODO(srawlins): Rename the parameters or refactor this code to avoid the + // confusion of `null` values for the "first*" parameters and non-`null` + // values for the "second*" parameters. + /// + /// If the reference contains three identifiers, each separated by a period, + /// then pass the associated Tokens as [firstToken], [firstPeriod], + /// [secondToken], [secondPeriod], and [identifierOrOperator]. + CommentReferenceImpl _parseOneCommentReferenceRest( + Token begin, + int referenceOffset, + Token? newKeyword, + Token? firstToken, + Token? firstPeriod, + Token? secondToken, + Token? secondPeriod, + Token identifierOrOperator, + ) { + // Adjust the token offsets to match the enclosing comment token. + var token = begin; + do { + token.offset += referenceOffset; + token = token.next!; + } while (!token.isEof); + + var identifier = SimpleIdentifierImpl(identifierOrOperator); + if (firstToken != null) { + var target = PrefixedIdentifierImpl( + prefix: SimpleIdentifierImpl(firstToken), + period: firstPeriod!, + identifier: SimpleIdentifierImpl(secondToken!), + ); + var expression = PropertyAccessImpl( + target: target, + operator: secondPeriod!, + propertyName: identifier, + ); + return CommentReferenceImpl( + newKeyword: newKeyword, + expression: expression, + ); + } else if (secondToken != null) { + var expression = PrefixedIdentifierImpl( + prefix: SimpleIdentifierImpl(secondToken), + period: secondPeriod!, + identifier: identifier, + ); + return CommentReferenceImpl( + newKeyword: newKeyword, + expression: expression, + ); + } else { + return CommentReferenceImpl( + newKeyword: newKeyword, + expression: identifier, + ); + } + } +} diff --git a/pkg/analyzer/test/generated/parser_test_base.dart b/pkg/analyzer/test/generated/parser_test_base.dart index d2909a37e5bc..1de892ddffc4 100644 --- a/pkg/analyzer/test/generated/parser_test_base.dart +++ b/pkg/analyzer/test/generated/parser_test_base.dart @@ -817,8 +817,7 @@ class ParserProxy extends analyzer.Parser { }); } - List parseCommentReferences( - List tokens) { + Comment parseComment(List tokens) { for (int index = 0; index < tokens.length - 1; ++index) { var next = tokens[index].next; if (next == null) { @@ -827,14 +826,13 @@ class ParserProxy extends analyzer.Parser { expect(next, tokens[index + 1]); } } - expect(tokens[tokens.length - 1].next, isNull); - List references = - astBuilder.parseCommentReferences(tokens.first); + expect(tokens.last.next, isNull); + var comment = astBuilder.parseDocComment(tokens.first); if (astBuilder.stack.isNotEmpty) { throw 'Expected empty stack, but found:' '\n ${astBuilder.stack.values.join('\n ')}'; } - return references; + return comment; } @override diff --git a/pkg/analyzer/test/generated/simple_parser_test.dart b/pkg/analyzer/test/generated/simple_parser_test.dart index ed327920be57..a5c7b35c5c1e 100644 --- a/pkg/analyzer/test/generated/simple_parser_test.dart +++ b/pkg/analyzer/test/generated/simple_parser_test.dart @@ -571,8 +571,8 @@ class C {} DocumentationCommentToken docToken = DocumentationCommentToken( TokenType.MULTI_LINE_COMMENT, "/** [ some text", 5); createParser(''); - List references = - parser.parseCommentReferences([docToken]); + var comment = parser.parseComment([docToken]); + var references = comment.references; expectNotNullIfNoErrors(references); assertNoErrors(); expect(references, hasLength(1)); diff --git a/pkg/analyzer/test/src/dart/parser/doc_comment_test.dart b/pkg/analyzer/test/src/dart/parser/doc_comment_test.dart index 3015ead089ef..a5f19720b038 100644 --- a/pkg/analyzer/test/src/dart/parser/doc_comment_test.dart +++ b/pkg/analyzer/test/src/dart/parser/doc_comment_test.dart @@ -14,7 +14,7 @@ main() { @reflectiveTest class DocCommentParserTest extends ParserDiagnosticsTest { - test_code() { + test_codeSpan() { final parseResult = parseStringWithErrors(r''' /// `a[i]` and [b]. class A {} @@ -34,7 +34,7 @@ Comment '''); } - test_code_legacy_block() { + test_codeSpan_legacy_blockComment() { // TODO(srawlins): I believe we should drop support for `[:` `:]`. final parseResult = parseStringWithErrors(r''' /** [:xxx [a] yyy:] [b] zzz */ @@ -54,7 +54,7 @@ Comment '''); } - test_code_unterminated() { + test_codeSpan_unterminated_blockComment() { final parseResult = parseStringWithErrors(r''' /** `a[i] and [b] */ class A {} @@ -76,146 +76,7 @@ Comment '''); } - test_codeBlock_backticks() { - final parseResult = parseStringWithErrors(r''' -/// First. -/// ```dart -/// a[i] = b[i]; -/// ``` -/// Last. -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('a[i]'); - // TODO(srawlins): Parse a backtick code block into its own node. - assertParsedNodeText(node, r''' -Comment - tokens - /// First. - /// ```dart - /// a[i] = b[i]; - /// ``` - /// Last. -'''); - } - - test_codeBlock_backticks_block() { - final parseResult = parseStringWithErrors(r''' -/** - * First. - * ```dart - * a[i] = b[i]; - * ``` - * Last. - */ -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('a[i]'); - // TODO(srawlins): Parse a backtick code block into its own node. - assertParsedNodeText(node, r''' -Comment - tokens - /** - * First. - * ```dart - * a[i] = b[i]; - * ``` - * Last. - */ -'''); - } - - test_codeBlock_indented_afterBlankLine() { - final parseResult = parseStringWithErrors(r''' -/// Text. -/// -/// a[i] = b[i]; -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('Text'); - // TODO(srawlins): Parse a backtick code block into its own node. - assertParsedNodeText(node, r''' -Comment - tokens - /// Text. - /// - /// a[i] = b[i]; -'''); - } - - test_codeBlock_indented_afterTextLine_notCodeBlock() { - final parseResult = parseStringWithErrors(r''' -/// Text. -/// a[i] = b[i]; -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('Text'); - // TODO(srawlins): Parse an indented code block into its own node. - assertParsedNodeText(node, r''' -Comment - references - CommentReference - expression: SimpleIdentifier - token: i - CommentReference - expression: SimpleIdentifier - token: i - tokens - /// Text. - /// a[i] = b[i]; -'''); - } - - test_codeBlock_indented_firstLine() { - final parseResult = parseStringWithErrors(r''' -/// a[i] = b[i]; -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('a[i]'); - // TODO(srawlins): Parse an indented code block into its own node. - assertParsedNodeText(node, r''' -Comment - tokens - /// a[i] = b[i]; -'''); - } - - test_codeBlock_indented_firstLine_block() { - final parseResult = parseStringWithErrors(r''' -/** - * a[i] = b[i]; - * [c]. - */ -class A {} -'''); - parseResult.assertNoErrors(); - - final node = parseResult.findNode.comment('a[i]'); - // TODO(srawlins): Parse an indented code block into its own node. - assertParsedNodeText(node, r''' -Comment - references - CommentReference - expression: SimpleIdentifier - token: c - tokens - /** - * a[i] = b[i]; - * [c]. - */ -'''); - } - - test_commentReference_block() { + test_commentReference_blockComment() { final parseResult = parseStringWithErrors(r''' /** [a]. */ class A {} @@ -275,7 +136,7 @@ Comment '''); } - test_commentReference_multiple_block() { + test_commentReference_multiple_blockComment() { final parseResult = parseStringWithErrors(r''' /** [a] and [b]. */ class A {} @@ -484,6 +345,275 @@ Comment '''); } + test_fencedCodeBlock_blockComment() { + final parseResult = parseStringWithErrors(r''' +/** + * One. + * ``` + * a[i] = b[i]; + * ``` + * Two. + * ```dart + * code; + * ``` + * Three. + */ +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + assertParsedNodeText(node, r''' +Comment + tokens + /** + * One. + * ``` + * a[i] = b[i]; + * ``` + * Two. + * ```dart + * code; + * ``` + * Three. + */ + fencedCodeBlocks + MdFencedCodeBlock + infoString: + lines + MdFencedCodeBlockLine + offset: 15 + length: 3 + MdFencedCodeBlockLine + offset: 22 + length: 12 + MdFencedCodeBlockLine + offset: 38 + length: 3 + MdFencedCodeBlock + infoString: dart + lines + MdFencedCodeBlockLine + offset: 53 + length: 7 + MdFencedCodeBlockLine + offset: 64 + length: 5 + MdFencedCodeBlockLine + offset: 73 + length: 3 +'''); + } + + test_fencedCodeBlock_nonDocCommentLines() { + final parseResult = parseStringWithErrors(r''' +/// One. +/// ``` +// This is not part of the doc comment. +/// a[i] = b[i]; + +/// ``` +/// Two. +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + assertParsedNodeText(node, r''' +Comment + tokens + /// One. + /// ``` + /// a[i] = b[i]; + /// ``` + /// Two. + fencedCodeBlocks + MdFencedCodeBlock + infoString: + lines + MdFencedCodeBlockLine + offset: 12 + length: 4 + MdFencedCodeBlockLine + offset: 60 + length: 13 + MdFencedCodeBlockLine + offset: 78 + length: 4 +'''); + } + + test_fencedCodeBlock_nonTerminating() { + final parseResult = parseStringWithErrors(r''' +/// One. +/// ``` +/// a[i] = b[i]; +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + assertParsedNodeText(node, r''' +Comment + tokens + /// One. + /// ``` + /// a[i] = b[i]; + fencedCodeBlocks + MdFencedCodeBlock + infoString: + lines + MdFencedCodeBlockLine + offset: 12 + length: 4 + MdFencedCodeBlockLine + offset: 20 + length: 13 +'''); + } + + test_fencedCodeBlocks() { + final parseResult = parseStringWithErrors(r''' +/// One. +/// ``` +/// a[i] = b[i]; +/// ``` +/// Two. +/// ```dart +/// code; +/// ``` +/// Three. +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + assertParsedNodeText(node, r''' +Comment + tokens + /// One. + /// ``` + /// a[i] = b[i]; + /// ``` + /// Two. + /// ```dart + /// code; + /// ``` + /// Three. + fencedCodeBlocks + MdFencedCodeBlock + infoString: + lines + MdFencedCodeBlockLine + offset: 12 + length: 4 + MdFencedCodeBlockLine + offset: 20 + length: 13 + MdFencedCodeBlockLine + offset: 37 + length: 4 + MdFencedCodeBlock + infoString: dart + lines + MdFencedCodeBlockLine + offset: 54 + length: 8 + MdFencedCodeBlockLine + offset: 66 + length: 6 + MdFencedCodeBlockLine + offset: 76 + length: 4 +'''); + } + + test_indentedCodeBlock_afterBlankLine() { + final parseResult = parseStringWithErrors(r''' +/// Text. +/// +/// a[i] = b[i]; +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('Text'); + assertParsedNodeText(node, r''' +Comment + tokens + /// Text. + /// + /// a[i] = b[i]; +'''); + } + + test_indentedCodeBlock_afterTextLine_notCodeBlock() { + final parseResult = parseStringWithErrors(r''' +/// Text. +/// a[i] = b[i]; +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('Text'); + // TODO(srawlins): Parse an indented code block into its own node. + assertParsedNodeText(node, r''' +Comment + references + CommentReference + expression: SimpleIdentifier + token: i + CommentReference + expression: SimpleIdentifier + token: i + tokens + /// Text. + /// a[i] = b[i]; +'''); + } + + test_indentedCodeBlock_firstLine() { + final parseResult = parseStringWithErrors(r''' +/// a[i] = b[i]; +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + // TODO(srawlins): Parse an indented code block into its own node. + assertParsedNodeText(node, r''' +Comment + tokens + /// a[i] = b[i]; +'''); + } + + test_indentedCodeBlock_firstLine_blockComment() { + final parseResult = parseStringWithErrors(r''' +/** + * a[i] = b[i]; + * [c]. + */ +class A {} +'''); + parseResult.assertNoErrors(); + + final node = parseResult.findNode.comment('a[i]'); + // TODO(srawlins): Parse an indented code block into its own node. + assertParsedNodeText(node, r''' +Comment + references + CommentReference + expression: SimpleIdentifier + token: c + tokens + /** + * a[i] = b[i]; + * [c]. + */ +'''); + } + test_inlineLink() { final parseResult = parseStringWithErrors(r''' /// [a](http://www.google.com) [b]. diff --git a/pkg/analyzer/test/src/fasta/ast_builder_test.dart b/pkg/analyzer/test/src/fasta/ast_builder_test.dart index c27b374ad6f6..9ad436181604 100644 --- a/pkg/analyzer/test/src/fasta/ast_builder_test.dart +++ b/pkg/analyzer/test/src/fasta/ast_builder_test.dart @@ -181,6 +181,19 @@ ClassDeclaration offset: 223 /// and [Object]. offset: 231 + fencedCodeBlocks + MdFencedCodeBlock + infoString: + lines + MdFencedCodeBlockLine + offset: 121 + length: 4 + MdFencedCodeBlockLine + offset: 129 + length: 36 + MdFencedCodeBlockLine + offset: 169 + length: 4 metadata Annotation atSign: @ @45 diff --git a/pkg/analyzer/test/src/summary/resolved_ast_printer.dart b/pkg/analyzer/test/src/summary/resolved_ast_printer.dart index c391d2226404..45584d547d82 100644 --- a/pkg/analyzer/test/src/summary/resolved_ast_printer.dart +++ b/pkg/analyzer/test/src/summary/resolved_ast_printer.dart @@ -253,6 +253,14 @@ class ResolvedAstPrinter extends ThrowingAstVisitor { _sink.writeln('Comment'); _sink.withIndent(() { _writeNamedChildEntities(node); + if (node.fencedCodeBlocks.isNotEmpty) { + _sink.writelnWithIndent('fencedCodeBlocks'); + _sink.withIndent(() { + for (var fencedCodeBlock in node.fencedCodeBlocks) { + _writeMdFencedCodeBlock(fencedCodeBlock); + } + }); + } }); } @@ -1702,6 +1710,25 @@ Expected parent: (${parent.runtimeType}) $parent } } + void _writeMdFencedCodeBlock(MdFencedCodeBlock fencedCodeBlock) { + _sink.writelnWithIndent('MdFencedCodeBlock'); + _sink.withIndent(() { + var infoString = fencedCodeBlock.infoString; + _sink.writelnWithIndent('infoString: ${infoString ?? ''}'); + assert(fencedCodeBlock.lines.isNotEmpty); + _sink.writelnWithIndent('lines'); + _sink.withIndent(() { + for (var line in fencedCodeBlock.lines) { + _sink.writelnWithIndent('MdFencedCodeBlockLine'); + _sink.withIndent(() { + _sink.writelnWithIndent('offset: ${line.offset}'); + _sink.writelnWithIndent('length: ${line.length}'); + }); + } + }); + }); + } + void _writeNamedChildEntities(AstNode node) { node as AstNodeImpl; for (var entity in node.namedChildEntities) {