Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

524 lines (479 sloc) 14.769 kB
package scalariform.lexer
import scala.annotation._
import scalariform.lexer.CharConstants.SU
import scalariform.lexer.Chars._
import scalariform.lexer.ScalaLexer._
import scalariform.lexer.Tokens._
import scalariform.utils.Utils
import scalariform._
/**
* Lexer implementation for non-XML Scala
*/
private[lexer] trait ScalaOnlyLexer { self: ScalaLexer
private var processingSymbol = false
private var possibleInterpolationId = false
protected def fetchScalaToken() {
(ch: @switch) match {
case ' ' | '\t' | '\n' | '\r' /* TODO: | FF */
nextChar()
getWhitespaceRest()
case 'A' | 'B' | 'C' | 'D' | 'E' |
'F' | 'G' | 'H' | 'I' | 'J' |
'K' | 'L' | 'M' | 'N' | 'O' |
'P' | 'Q' | 'R' | 'S' | 'T' |
'U' | 'V' | 'W' | 'X' | 'Y' |
'Z' | '$' | '_' |
'a' | 'b' | 'c' | 'd' | 'e' |
'f' | 'g' | 'h' | 'i' | 'j' |
'k' | 'l' | 'm' | 'n' | 'o' |
'p' | 'q' | 'r' | 's' | 't' |
'u' | 'v' | 'w' | 'x' | 'y' |
'z'
nextChar()
try {
possibleInterpolationId = true
getIdentRest()
} finally
possibleInterpolationId = false
case '<'
lastCh match {
case SU | ' ' | '\t' | '\n' | '{' | '(' | '>' if ch(1) != SU && (isNameStart(ch(1)) || ch(1) == '!' || ch(1) == '?')
switchToXmlModeAndFetchToken()
case _
nextChar()
getOperatorRest()
}
case '~' | '!' | '@' | '#' | '%' |
'^' | '*' | '+' | '-' | /*'<' | */
'>' | '?' | ':' | '=' | '&' |
'|' | '\\'
nextChar()
getOperatorRest()
case '/'
(ch(1): @switch) match {
case '/' getSingleLineComment()
case '*' getMultilineComment()
case _ nextChar(); getOperatorRest()
}
case '0'
if (ch(1) == 'x' || ch(1) == 'X')
getHexNumber()
else {
nextChar()
getNumber(base = 8)
}
case '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
getNumber(base = 10)
case '`' getBackquotedIdent()
case '"'
if (ch(1) == '"' && ch(2) == '"')
getMultiLineStringLit()
else
getStringLit()
case '\''
nextChar()
if (isIdentifierStart(ch))
charLitOr(getIdentRest)
else if (isOperatorPart(ch) && (ch != '\\'))
charLitOr(getOperatorRest)
else {
getLitChar()
if (ch == '\'' || forgiveErrors) {
nextChar()
token(CHARACTER_LITERAL)
} else
throw new ScalaLexerException("unclosed character literal")
}
case '.'
nextChar()
if ('0' <= ch && ch <= '9')
getFraction()
else
token(DOT)
case ';'
nextChar(); token(SEMI)
case ','
nextChar(); token(COMMA)
case '('
nextChar(); token(LPAREN)
case '{'
nextChar(); token(LBRACE)
scalaMode.nestBrace()
case ')'
nextChar(); token(RPAREN)
case '}'
nextChar(); token(RBRACE)
val nestingLevel = scalaMode.unnestBrace()
if (nestingLevel == 0 && !isRootMode)
popMode() // Go back to XML or string interpolation
case '['
nextChar(); token(LBRACKET)
case ']'
nextChar(); token(RBRACKET)
case SU
token(EOF)
case _
if (ch == '\u21D2') {
nextChar()
token(ARROW)
} else if (ch == '\u2190') {
nextChar()
token(LARROW)
} else if (Character.isUnicodeIdentifierStart(ch)) {
nextChar()
getIdentRest()
} else if (isSpecial(ch)) {
nextChar()
getOperatorRest()
} else if (forgiveErrors) {
nextChar()
getWhitespaceRest()
} else
throw new ScalaLexerException("illegal character: " + ch)
}
}
@tailrec
private def getWhitespaceRest(): Unit = (ch: @switch) match {
case ' ' | '\t' | '\n' | '\r' /* TODO: | FF */
nextChar()
getWhitespaceRest()
case _
token(WS)
}
private def getStringLit() {
getStringLitOrBackquotedIdent(delimiter = '"', errorMsg = "unclosed string literal", tokenType = STRING_LITERAL)
}
private def getBackquotedIdent() {
getStringLitOrBackquotedIdent(delimiter = '`', errorMsg = "unclosed quoted identifer", tokenType = VARID, errorMsgOnEmpty = Some("empty quoted identifier"))
}
private def getStringLitOrBackquotedIdent(delimiter: Char, errorMsg: String, tokenType: TokenType, errorMsgOnEmpty: Option[String] = None) {
// require(ch == delimiter)
nextChar()
@tailrec
def scanForClosingQuotes(firstTime: Boolean): Unit = ch match {
case _ if eof
if (forgiveErrors) token(tokenType) else throw new ScalaLexerException(errorMsg)
case '\r' | '\n' if !isUnicodeEscape
if (forgiveErrors) token(tokenType) else throw new ScalaLexerException(errorMsg)
case `delimiter` if firstTime && errorMsgOnEmpty.isDefined
if (forgiveErrors) token(tokenType) else throw new ScalaLexerException(errorMsgOnEmpty.get)
case `delimiter`
nextChar()
token(tokenType)
case _
getLitChar()
scanForClosingQuotes(firstTime = false)
}
scanForClosingQuotes(firstTime = true)
}
private def getLitChar() {
if (ch == '\\') {
nextChar()
if ('0' <= ch && ch <= '7') {
val leadch = ch
nextChar()
if ('0' <= ch && ch <= '7') {
nextChar()
if (leadch <= '3' && '0' <= ch && ch <= '7')
nextChar()
}
} else (ch: @switch) match {
case 'b' | 't' | 'n' | 'f' | 'r' | '"' | '\'' | '\\' nextChar()
case _ if (forgiveErrors) nextChar() else throw new ScalaLexerException("invalid escape character")
}
} else
nextChar()
}
private def getMultiLineStringLit() {
munch("\"\"\"")
@tailrec
def scanForClosingTripleQuotes() {
if (lookaheadIs("\"\"\"")) {
munch("\"\"\"")
while (ch == '\"') { nextChar() }
token(STRING_LITERAL)
} else if (eof) {
if (forgiveErrors) token(STRING_LITERAL) else throw new ScalaLexerException("unclosed multi-line string literal")
} else {
nextChar()
scanForClosingTripleQuotes()
}
}
scanForClosingTripleQuotes()
}
@tailrec
final protected def getStringPart(multiLine: Boolean) {
if (ch == '"') {
if (multiLine) {
nextChar()
if (isTripleQuote()) {
token(STRING_LITERAL)
popMode()
} else
getStringPart(multiLine)
} else {
nextChar()
token(STRING_LITERAL)
popMode()
}
} else if (ch == '$') {
nextChar()
if (ch == '$') {
nextChar()
getStringPart(multiLine)
} else if (ch == '{') {
token(STRING_PART)
switchToScalaMode()
} else if (Character.isUnicodeIdentifierStart(ch)) {
token(STRING_PART)
stringInterpolationMode.interpolationVariable = true
} else {
if (forgiveErrors) {
nextChar()
getStringPart(multiLine)
} else
throw new ScalaLexerException("invalid string interpolation")
}
} else {
val isUnclosedLiteral = !isUnicodeEscape && (ch == SU || (!multiLine && (ch == '\r' || ch == '\n')))
if (isUnclosedLiteral) {
if (forgiveErrors) {
token(STRING_LITERAL)
popMode()
} else
throw new ScalaLexerException(if (!multiLine) "unclosed string literal" else "unclosed multi-line string literal")
} else {
nextChar()
getStringPart(multiLine)
}
}
}
private def isTripleQuote(): Boolean =
if (ch == '"') {
nextChar()
if (ch == '"') {
nextChar()
while (ch == '"')
nextChar()
true
} else
false
} else
false
private def getIdentRest(): Unit = (ch: @switch) match {
case 'A' | 'B' | 'C' | 'D' | 'E' |
'F' | 'G' | 'H' | 'I' | 'J' |
'K' | 'L' | 'M' | 'N' | 'O' |
'P' | 'Q' | 'R' | 'S' | 'T' |
'U' | 'V' | 'W' | 'X' | 'Y' |
'Z' | '$' |
'a' | 'b' | 'c' | 'd' | 'e' |
'f' | 'g' | 'h' | 'i' | 'j' |
'k' | 'l' | 'm' | 'n' | 'o' |
'p' | 'q' | 'r' | 's' | 't' |
'u' | 'v' | 'w' | 'x' | 'y' |
'z' |
'0' | '1' | '2' | '3' | '4' |
'5' | '6' | '7' | '8' | '9'
nextChar()
getIdentRest()
case '_'
nextChar()
getIdentOrOperatorRest()
case _
if (Character.isUnicodeIdentifierPart(ch) && ch != SU) {
nextChar()
getIdentRest()
} else
finishNamed()
}
private def getOperatorRest(): Unit = (ch: @switch) match {
case '~' | '!' | '@' | '#' | '%' |
'^' | '*' | '+' | '-' | '<' |
'>' | '?' | ':' | '=' | '&' |
'|' | '\\'
nextChar(); getOperatorRest()
case '/'
(ch(1): @switch) match {
case '/' | '*' finishNamed()
case _ nextChar(); getOperatorRest()
}
case _
if (isSpecial(ch)) { nextChar(); getOperatorRest() } else finishNamed()
}
private def getIdentOrOperatorRest() {
if (isIdentifierPart(ch))
getIdentRest()
else ch match {
case '~' | '!' | '@' | '#' | '%' |
'^' | '*' | '+' | '-' | '<' |
'>' | '?' | ':' | '=' | '&' |
'|' | '\\' | '/'
getOperatorRest()
case _
if (isSpecial(ch))
getOperatorRest()
else
finishNamed()
}
}
private def finishNamed() {
val tokenType =
if (processingSymbol)
SYMBOL_LITERAL
else if (possibleInterpolationId && ch == '\"' && scalaVersion >= ScalaVersions.Scala_2_10) {
switchToStringInterpolationMode(lookaheadIs("\"\"\""))
INTERPOLATION_ID
} else
Keywords(getTokenText).getOrElse(VARID)
token(tokenType)
}
private def getSingleLineComment() {
// require(ch == '/')
nextChar()
// require(ch == '/')
nextChar()
@tailrec
def consumeUntilNewline(): Unit = (ch: @switch) match {
case '\n'
nextChar()
token(LINE_COMMENT)
case '\r' if ch(1) != '\n'
nextChar()
token(LINE_COMMENT)
case SU if eof
token(LINE_COMMENT)
case _
nextChar()
consumeUntilNewline()
}
consumeUntilNewline()
}
private def getMultilineComment() {
munch("/*")
@tailrec
def consumeUntilSplatSlash(nesting: Int) {
if (nesting == 0)
token(MULTILINE_COMMENT)
else
(ch: @switch) match {
case '*' if (ch(1) == '/')
nextChar()
nextChar()
consumeUntilSplatSlash(nesting - 1)
case '/' if (ch(1) == '*')
nextChar()
nextChar()
consumeUntilSplatSlash(nesting + 1)
case SU if eof
if (forgiveErrors) token(MULTILINE_COMMENT) else throw new ScalaLexerException("Unterminated comment")
case _
nextChar()
consumeUntilSplatSlash(nesting)
}
}
consumeUntilSplatSlash(nesting = 1)
}
private def getFraction() {
while ('0' <= ch && ch <= '9') { nextChar() }
if (ch == 'e' || ch == 'E') {
nextChar()
if (ch == '+' || ch == '-')
nextChar()
while ('0' <= ch && ch <= '9') { nextChar() }
}
if (ch == 'd' || ch == 'D' || ch == 'f' || ch == 'F')
nextChar()
checkNoLetter()
token(FLOATING_POINT_LITERAL)
}
private def getHexNumber() {
// require(ch == '0')
nextChar()
// require(ch == 'x' || ch == 'X')
nextChar()
@tailrec
def munchHexDigits(): Unit = (ch: @switch) match {
case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F'
nextChar(); munchHexDigits()
case 'l' | 'L'
nextChar(); token(INTEGER_LITERAL)
case _
token(INTEGER_LITERAL); checkNoLetter()
}
munchHexDigits()
}
private def getNumber(base: Int) {
def isDigit(c: Char) = if (c == SU) false else (Character isDigit c)
val base1 = if (base < 10) 10 else base
// read 8,9's even if format is octal, produce a malformed number error afterwards.
while (Utils.digit2int(ch, base1) >= 0)
nextChar()
def restOfUncertainToken() = {
def isEfd = ch match {
case 'e' | 'E' | 'f' | 'F' | 'd' | 'D' true
case _ false
}
def isL = ch match {
case 'l' | 'L' true
case _ false
}
if (isEfd)
getFraction()
else {
if (isL)
nextChar()
else
checkNoLetter()
token(INTEGER_LITERAL)
}
}
if (ch == '.') {
val c = ch(1)
if (scalaVersion >= ScalaVersions.Scala_2_11 && !isDigit(c)) {
token(INTEGER_LITERAL)
return
}
val isDefinitelyNumber =
(c: @switch) match {
/** Another digit is a giveaway. */
case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
true
/** Backquoted idents like 22.`foo`. */
case '`'
token(INTEGER_LITERAL); return
/** Note the early return */
/** These letters may be part of a literal, or a method invocation on an Int */
case 'd' | 'D' | 'f' | 'F'
!isIdentifierPart(ch(2))
/** A little more special handling for e.g. 5e7 */
case 'e' | 'E'
val ch2 = ch(2)
!isIdentifierPart(ch2) || (isDigit(ch2) || ch2 == '+' || ch2 == '-')
case x
!isIdentifierStart(x)
}
if (isDefinitelyNumber) {
nextChar()
getFraction()
} else
restOfUncertainToken()
} else
restOfUncertainToken()
}
private def checkNoLetter() {
if (isIdentifierPart(ch) && ch >= ' ' && !forgiveErrors)
throw new ScalaLexerException("Invalid literal number: " + ch)
}
private def charLitOr(op: () Unit) {
nextChar()
if (ch == '\'') {
nextChar()
token(CHARACTER_LITERAL)
} else {
processingSymbol = true
op()
processingSymbol = false
}
}
}
Jump to Line
Something went wrong with that request. Please try again.