Skip to content

Commit

Permalink
Fix rustdoc whitespace parsing
Browse files Browse the repository at this point in the history
Previously, whitespace on the second line of this fragment
was incorrectly parsed as 2
`PsiWhiteSpace` elements (" " and "\n").
Now it is parsed as a single " \n" whitespace element.

Replace `.` to a space

```
///.Extra space here:
///.
///
fn foo() {}
```
  • Loading branch information
vlad20012 committed Jun 18, 2021
1 parent bf12f51 commit 39a9e17
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 27 deletions.
60 changes: 33 additions & 27 deletions src/main/kotlin/org/rust/lang/doc/psi/RsDocCommentElementType.kt
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ import com.intellij.util.text.CharArrayUtil
import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.parser.MarkdownParser
import org.rust.lang.RsLanguage
import org.rust.lang.doc.psi.RsDocElementTypes.DOC_DATA
import org.rust.lang.doc.psi.RsDocElementTypes.DOC_GAP
import org.rust.lang.doc.psi.impl.RsDocCommentImpl
import org.rust.lang.doc.psi.impl.RsDocGapImpl
import org.rust.stdext.exhaustive
import kotlin.math.max
import kotlin.math.min

Expand Down Expand Up @@ -82,26 +85,13 @@ private class RsDocMarkdownAstBuilder(

private fun CompositeElement.insertLeaves(startOffset: Int, endOffset: Int) {
textMap.processPiecesInRange(startOffset, endOffset) { piece ->
when (piece.kind) {
PieceKind.TEXT -> {
rawAddChildrenWithoutNotifications(LeafPsiElement(RsDocElementTypes.DOC_DATA, charTable.intern(piece.str)))
}
PieceKind.GAP -> {
val gapStart = CharArrayUtil.shiftForward(piece.str, 0, "\n\t ")
if (gapStart != 0) {
rawAddChildrenWithoutNotifications(PsiWhiteSpaceImpl(charTable.intern(piece.str.substring(0, gapStart))))
}
if (gapStart != piece.str.length) {
val gapEnd = CharArrayUtil.shiftBackward(piece.str, gapStart, piece.str.lastIndex, "\n\t ") + 1
val gapText = charTable.intern(piece.str, gapStart, gapEnd)
check(gapText.isNotEmpty())
rawAddChildrenWithoutNotifications(RsDocGapImpl(RsDocElementTypes.DOC_GAP, gapText))
if (gapEnd != piece.str.length) {
rawAddChildrenWithoutNotifications(PsiWhiteSpaceImpl(charTable.intern(piece.str.substring(gapEnd))))
}
}
}
val internedText = charTable.intern(piece.str)
val element = when (piece.kind) {
PieceKind.TEXT -> LeafPsiElement(DOC_DATA, internedText)
PieceKind.GAP -> RsDocGapImpl(DOC_GAP, internedText)
PieceKind.WS -> PsiWhiteSpaceImpl(internedText)
}
rawAddChildrenWithoutNotifications(element)
}
}

Expand Down Expand Up @@ -143,7 +133,7 @@ private class RsDocTextMap(
if (line.hasPrefix) {
val prefix = line.prefix
textPosition += prefix.length
pieces.mergeOrAddGap(prefix)
pieces.mergeOrAddGapWithWS(prefix)
}

if (line.hasContent) {
Expand All @@ -167,13 +157,13 @@ private class RsDocTextMap(
mappedText.append("\n")
}
textPosition += 1
pieces += Piece("\n", PieceKind.GAP)
pieces.mergeOrAddWS("\n")
}

if (line.hasSuffix) {
val suffix = line.suffix
textPosition += suffix.length
pieces.mergeOrAddGap(suffix)
pieces.mergeOrAddGapWithWS(suffix)
}
}

Expand All @@ -184,19 +174,35 @@ private class RsDocTextMap(
return RsDocTextMap(text, mappedText, map, pieces)
}

private fun MutableList<Piece>.mergeOrAddGap(gap: CharSequence) {
if (lastOrNull()?.kind == PieceKind.GAP) {
this[lastIndex] = Piece(this[lastIndex].str.toString() + gap, PieceKind.GAP)
private fun MutableList<Piece>.mergeOrAddGapWithWS(str: CharSequence) {
val gapStart = CharArrayUtil.shiftForward(str, 0, "\n\t ")
if (gapStart != 0) {
mergeOrAddWS(str.subSequence(0, gapStart))
}
if (gapStart != str.length) {
val gapEnd = CharArrayUtil.shiftBackward(str, gapStart, str.lastIndex, "\n\t ") + 1
val gapText = str.subSequence(gapStart, gapEnd)
check(gapText.isNotEmpty())
this += Piece(gapText, PieceKind.GAP)
if (gapEnd != str.length) {
mergeOrAddWS(str.subSequence(gapEnd, str.length))
}
}
}

private fun MutableList<Piece>.mergeOrAddWS(gap: CharSequence) {
if (lastOrNull()?.kind == PieceKind.WS) {
this[lastIndex] = Piece(this[lastIndex].str.toString() + gap, PieceKind.WS)
} else {
this += Piece(gap, PieceKind.GAP)
this += Piece(gap, PieceKind.WS)
}
}
}
}

private data class Piece(val str: CharSequence, val kind: PieceKind)

private enum class PieceKind { TEXT, GAP }
private enum class PieceKind { TEXT, GAP, WS }

private fun Piece.cut(startOffset: Int, endOffset: Int): Piece {
val newStr = str.subSequence(max(0, startOffset), min(endOffset, str.length))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class RsCompleteParsingTestCase : RsParsingTestCaseBase("complete") {
fun `test andand`() = doTest(true)
fun `test comment binding`() = doTest(true)
fun `test doc comments`() = doTest(true)
fun `test doc comment whitespace`() = doTest(true)
fun `test associated types`() = doTest(true)
fun `test last block is expression`() = doTest(true)
fun `test loops`() = doTest(true)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
/// Extra space here:
///
///
fn foo() {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
FILE
RsFunctionImpl(FUNCTION)
PsiComment(<OUTER_EOL_DOC_COMMENT>)
PsiElement(<DOC_GAP>)('///')
PsiWhiteSpace(' ')
PsiElement(<DOC_DATA>)('Extra space here:')
PsiWhiteSpace('\n')
PsiElement(<DOC_GAP>)('///')
PsiWhiteSpace(' \n')
PsiElement(<DOC_GAP>)('///')
PsiWhiteSpace('\n')
PsiElement(fn)('fn')
PsiWhiteSpace(' ')
PsiElement(identifier)('foo')
RsValueParameterListImpl(VALUE_PARAMETER_LIST)
PsiElement(()('(')
PsiElement())(')')
PsiWhiteSpace(' ')
RsBlockImpl(BLOCK)
PsiElement({)('{')
PsiElement(})('}')

0 comments on commit 39a9e17

Please sign in to comment.