Merge pull request #3097 from kitbellew/3095

LegacyScanner: treat comment chars differently
scalameta · Mar 16, 2023 · e01a87d · e01a87d
2 parents 9e5c7d8 + 70e33ca
commit e01a87d
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 11 deletions.
diff --git a/...eta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala b/...eta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
@@ -55,6 +55,16 @@ class CharArrayReader(input: Input, dialect: Dialect, reporter: Reporter)
     }
   }
 
+  final def nextCommentChar(): Unit = {
+    if (charOffset >= buf.length) {
+      ch = SU
+    } else {
+      ch = buf(charOffset)
+      charOffset += 1
+      checkLineEnd()
+    }
+  }
+
   /**
    * Advance one character, leaving CR;LF pairs intact. This is for use in multi-line strings, so
    * there are no "potential line ends" here.
@@ -132,13 +142,18 @@ class CharArrayReader(input: Input, dialect: Dialect, reporter: Reporter)
 
   /** Handle line ends */
   private def potentialLineEnd(): Unit = {
-    if (ch == LF || ch == FF) {
-      if (!dialect.allowMultilinePrograms) {
-        readerError("line breaks are not allowed in single-line quasiquotes", at = charOffset - 1)
-      }
+    if (checkLineEnd() && !dialect.allowMultilinePrograms) {
+      readerError("line breaks are not allowed in single-line quasiquotes", at = charOffset - 1)
+    }
+  }
+
+  private def checkLineEnd(): Boolean = {
+    val ok = ch == LF || ch == FF
+    if (ok) {
       lastLineStartOffset = lineStartOffset
       lineStartOffset = charOffset
     }
+    ok
   }
 
   /** A new reader that takes off at the current character position */

diff --git a/...ameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala b/...ameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
@@ -23,13 +23,13 @@ class LegacyScanner(input: Input, dialect: Dialect) {
 
   private def isDigit(c: Char) = java.lang.Character isDigit c
   private var openComments = 0
-  protected def putCommentChar(): Unit = nextChar()
+  protected def putCommentChar(): Unit = nextCommentChar()
 
   @tailrec private def skipLineComment(): Unit = ch match {
     case SU | CR | LF =>
     case '$' if isUnquoteNextNoDollar() =>
       syntaxError("can't unquote into single-line comments", at = charOffset - 1)
-    case _ => nextChar(); skipLineComment()
+    case _ => putCommentChar(); skipLineComment()
   }
   private def maybeOpen(): Unit = {
     putCommentChar()
@@ -41,9 +41,10 @@ class LegacyScanner(input: Input, dialect: Dialect) {
   private def maybeClose(): Boolean = {
     putCommentChar()
     (ch == '/') && {
-      putCommentChar()
       openComments -= 1
-      openComments == 0
+      val close = openComments == 0
+      if (close) nextChar() else putCommentChar()
+      close
     }
   }
   @tailrec final def skipNestedComments(): Unit = ch match {
@@ -58,12 +59,12 @@ class LegacyScanner(input: Input, dialect: Dialect) {
   def skipBlockComment(): Unit = skipNestedComments()
 
   private def skipToCommentEnd(isLineComment: Boolean): Unit = {
-    nextChar()
+    putCommentChar()
     if (isLineComment) skipLineComment()
     else {
       openComments = 1
-      val isDocComment = (ch == '*') && { nextChar(); true }
-      if (isDocComment) {
+      if (ch == '*') {
+        putCommentChar()
         // Check for the amazing corner case of /**/
         if (ch == '/')
           nextChar()

diff --git a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
@@ -654,6 +654,22 @@ class TokenizerSuite extends BaseTokenizerSuite {
     """.trim.stripMargin)
   }
 
+  test("showRaw with comments - skip unicode escape 1") {
+    val comment = "// Note: '\\u000A' = '\\n'"
+    assertNoDiff(
+      tokenize(comment).map(_.structure).mkString("\n"),
+      Seq("BOF [0..0)", s"$comment [0..24)", "EOF [24..24)").mkString("\n")
+    )
+  }
+
+  test("showRaw with comments - skip unicode escape 2") {
+    val comment = "/* Note: '\\u000A' = '\\n' */"
+    assertNoDiff(
+      tokenize(comment).map(_.structure).mkString("\n"),
+      Seq("BOF [0..0)", s"$comment [0..27)", "EOF [27..27)").mkString("\n")
+    )
+  }
+
   test("interpolation start & end - episode 01") {
     assert(tokenize("q\"\"").map(_.structure).mkString("\n") == """
       |BOF [0..0)