Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tokenizers.LegacyScanner: clean up #3332

Merged
merged 1 commit into from
Sep 13, 2023
Merged

tokenizers.LegacyScanner: clean up #3332

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
92 changes: 28 additions & 64 deletions ...ameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {

private def isDigit(c: Char) = java.lang.Character isDigit c
private var openComments = 0
protected def putCommentChar(): Unit = nextCommentChar()
private def putCommentChar(): Unit = nextCommentChar()

@tailrec private def skipLineComment(): Unit = ch match {
case SU | CR | LF =>
Expand All @@ -47,18 +47,17 @@ class LegacyScanner(input: Input, dialect: Dialect) {
close
}
}
@tailrec final def skipNestedComments(): Unit = ch match {
@tailrec private final def skipNestedComments(): Unit = ch match {
case '/' => maybeOpen(); skipNestedComments()
case '*' => if (!maybeClose()) skipNestedComments()
case SU => incompleteInputError("unclosed comment", at = offset)
case '$' if isUnquoteNextNoDollar() =>
syntaxError("can't unquote into multi-line comments", at = charOffset - 1)
case _ => putCommentChar(); skipNestedComments()
}
def skipDocComment(): Unit = skipNestedComments()
def skipBlockComment(): Unit = skipNestedComments()

private def skipToCommentEnd(isLineComment: Boolean): Unit = {
private def skipToCommentEnd(): Unit = {
val isLineComment = ch == '/'
putCommentChar()
if (isLineComment) skipLineComment()
else {
Expand All @@ -69,54 +68,30 @@ class LegacyScanner(input: Input, dialect: Dialect) {
if (ch == '/')
nextChar()
else
skipDocComment()
} else skipBlockComment()
skipNestedComments()
} else skipNestedComments()
}
}

/**
* Precondition: ch == '/' Returns true if a comment was skipped.
*/
def skipComment(): Boolean = ch match {
case '/' | '*' => skipToCommentEnd(isLineComment = ch == '/'); true
case _ => false
}
def flushDoc(): Unit = ()

/**
* To prevent doc comments attached to expressions from leaking out of scope onto the next
* documentable entity, they are discarded upon passing a right brace, bracket, or parenthesis.
*/
def discardDocBuffer(): Unit = ()

def isAtEnd = charOffset >= buf.length

def resume(lastCode: LegacyToken) = {
token = lastCode
if (next.token != EMPTY)
syntaxError("unexpected end of input: possible missing '}' in XML block", at = offset)

nextToken()
}
private def isAtEnd = charOffset >= buf.length

/**
* A character buffer for literals
*/
val cbuf = new java.lang.StringBuilder
private val cbuf = new java.lang.StringBuilder

/**
* append Unicode character to "cbuf" buffer
*/
protected def putChar(c: Char): Unit = {
// assert(cbuf.size < 10000, cbuf)
private def putChar(c: Char): Unit = {
cbuf.append(c)
}

/**
* Determines whether this scanner should emit identifier deprecation warnings, e.g. when seeing
* `macro' or `then', which are planned to become keywords in future versions of Scala.
*/
protected def emitIdentifierDeprecationWarnings = true
private def emitIdentifierDeprecationWarnings = true

/** Clear buffer and set name and token */
private def finishNamed(isBackquoted: Boolean = false): Unit = {
Expand Down Expand Up @@ -159,7 +134,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
* - STRINGLIT if region is a string interpolation expression starting with '${' (the STRINGLIT
* appears twice in succession on the stack iff the expression is a multiline string literal).
*/
var sepRegions: List[LegacyToken] = List()
private var sepRegions: List[LegacyToken] = List()

@inline private def pushSepRegions(sr: LegacyToken) = sepRegions = sr :: sepRegions

Expand All @@ -176,7 +151,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
* The keys are offset start positions of an xml literal and the values are the respective offset
* end positions and a boolean indicating if the part is the last part.
*/
val upcomingXmlLiteralParts = mutable.Map.empty[Offset, (Offset, Boolean)]
private val upcomingXmlLiteralParts = mutable.Map.empty[Offset, (Offset, Boolean)]

// Get next token ------------------------------------------------------------

Expand All @@ -198,15 +173,6 @@ class LegacyScanner(input: Input, dialect: Dialect) {
popSepRegions()
}

/**
* read next token and return last offset
*/
def skipToken(): Offset = {
val off = offset
nextToken()
off
}

/**
* Produce next token, filling curr TokenData fields of Scanner.
*/
Expand All @@ -227,12 +193,8 @@ class LegacyScanner(input: Input, dialect: Dialect) {
popSepRegions()
if (sepRegions.nonEmpty)
popSepRegions()

discardDocBuffer()
case RBRACKET | RPAREN =>
if (isSepRegionsHead(lastToken)) popSepRegions()

discardDocBuffer()
case ARROW =>
if (isSepRegionsHead(lastToken)) popSepRegions()
case STRINGLIT =>
Expand Down Expand Up @@ -282,7 +244,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
/**
* read next token, filling TokenData fields of Scanner.
*/
protected final def fetchToken(): Unit = {
private final def fetchToken(): Unit = {
offset = charOffset - 1

if (inStringInterpolation) return getStringPart(multiLine = startsStringPart(sepRegions.tail))
Expand Down Expand Up @@ -347,11 +309,13 @@ class LegacyScanner(input: Input, dialect: Dialect) {
getOperatorRest()
case '/' =>
nextChar()
if (skipComment()) {
token = COMMENT
} else {
putChar('/')
getOperatorRest()
ch match {
case '/' | '*' =>
skipToCommentEnd()
token = COMMENT
case _ =>
putChar('/')
getOperatorRest()
}
case '0' =>
def fetchZero() = {
Expand Down Expand Up @@ -779,7 +743,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
nextChar()
}

protected def invalidEscape(): Unit = {
private def invalidEscape(): Unit = {
syntaxError("invalid escape character", at = charOffset - 1)
putChar(ch)
}
Expand All @@ -801,7 +765,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
else isSeparator
}

def checkNoTrailingSeparator(): Unit = {
private def checkNoTrailingSeparator(): Unit = {
val last = cbuf.length() - 1
if (last >= 0 && isNumberSeparator(cbuf.charAt(last))) {
syntaxError("trailing separator is not allowed", at = offset + last)
Expand All @@ -812,7 +776,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
/**
* read fractional part and exponent of floating point number if one is present.
*/
protected def getFraction(): Unit = {
private def getFraction(): Unit = {
while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
putChar(ch)
nextChar()
Expand Down Expand Up @@ -854,15 +818,15 @@ class LegacyScanner(input: Input, dialect: Dialect) {
setStrVal()
}

def checkNoLetter(): Unit = {
private def checkNoLetter(): Unit = {
if (isIdentifierPart(ch) && ch >= ' ' && !isNumberSeparator(ch))
syntaxError("Invalid literal number", at = offset)
}

/**
* Read a number into strVal and set base
*/
protected def getNumber(): Unit = {
private def getNumber(): Unit = {
val base1 = if (base < 10) 10 else base
// Read 8,9's even if format is octal, produce a malformed number error afterwards.
// At this point, we have already read the first digit, so to tell an innocent 0 apart
Expand Down Expand Up @@ -949,7 +913,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
* Parse character literal if current character is followed by \', or follow with given op and
* return a symbol literal token
*/
def charLitOr(op: () => Unit): Unit = {
private def charLitOr(op: () => Unit): Unit = {
putChar(ch)
nextChar()
if (ch == '\'') {
Expand All @@ -963,7 +927,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {
}
}

def getXml(): Unit = {
private def getXml(): Unit = {
// 1. Collect positions of scala expressions inside this xml literal.
import fastparse.Parsed
val start = offset
Expand Down Expand Up @@ -995,7 +959,7 @@ class LegacyScanner(input: Input, dialect: Dialect) {

// Unquotes -----------------------------------------------------------------

def getUnquote(): Unit = {
private def getUnquote(): Unit = {
require(ch == '$')
val start = charOffset
val endInclusive = {
Expand Down