Skip to content
Browse files

Support string interpolation literals and macro keywords in partition…

…er / syntax colouring. Fixed #1001012.
  • Loading branch information...
1 parent 93cf236 commit 7e1e01265324ff7c250f41809d04adb9076e1b0d @mdr mdr committed May 7, 2012
View
8 ...ala-ide.sdt.core.tests/src/scala/tools/eclipse/lexical/ScalaDocumentPartitionerTest.scala
@@ -8,12 +8,12 @@ import org.eclipse.jface.text._
class ScalaDocumentPartitionerTest {
@Test
- def no_partition_change {
+ def no_partition_change {
// 000000000011111111112222222222333333333344444444445
// 012345678901234567890123456789012345678901234567890
check("""/* comment */ "foo" /* comment */""", Replace(start = 5, finish = 7, text = "foo"), expectedNoRegion)
}
-
+
@Test
def modify_single_partition {
// 000000000011111111112222222222333333333344444444445
@@ -54,9 +54,9 @@ class ScalaDocumentPartitionerTest {
}
case class Replace(start: Int, finish: Int, text: String) extends Replacement {
- def docEvent(implicit doc: IDocument): DocumentEvent = new DocumentEvent(doc, start, finish - start + 1, text)
+ def docEvent(implicit doc: IDocument): DocumentEvent = new DocumentEvent(doc, start, finish - start + 1, text)
}
-
+
case class Deletion(start: Int, finish: Int) extends Replacement {
def docEvent(implicit doc: IDocument): DocumentEvent = new DocumentEvent(doc, start, finish - start + 1, "")
}
View
26 ...cala-ide.sdt.core.tests/src/scala/tools/eclipse/lexical/ScalaPartitionTokeniserTest.scala
@@ -76,6 +76,30 @@ class ScalaPartitionTokeniserTest {
<t>"""scala multiline string"""</t> ==> ((SCALA_MULTI_LINE_STRING, 0, 27))
}
+ @Test
+ def stringInterpolation {
+ // 000000000011111111112222222222333333333344444444445
+ // 012345678901234567890123456789012345678901234567890
+ <t>s"my name is $name"</t> ==>
+ ((DEFAULT_CONTENT_TYPE, 0, 0), (JAVA_STRING, 1, 13), (DEFAULT_CONTENT_TYPE, 14, 17), (JAVA_STRING, 18, 18))
+
+ // 000000000011111111112222222222333333333344444444445
+ // 012345678901234567890123456789012345678901234567890
+ <t>s"""my name is $name"""</t> ==>
+ ((DEFAULT_CONTENT_TYPE, 0, 0), (SCALA_MULTI_LINE_STRING, 1, 15), (DEFAULT_CONTENT_TYPE, 16, 19), (SCALA_MULTI_LINE_STRING, 20, 22))
+
+ // 000000000011111111112222222222333333333344444444445
+ // 012345678901234567890123456789012345678901234567890
+ """s"my name is ${person.name}"""" ==>
+ ((DEFAULT_CONTENT_TYPE, 0, 0), (JAVA_STRING, 1, 13), (DEFAULT_CONTENT_TYPE, 14, 26), (JAVA_STRING, 27, 27))
+
+ // 0 0 00000001111111111222222222 2 3 33333333344444444445
+ // 1 2 34567890123456789012345678 9 0 12345678901234567890
+ "s\"\"\"my name is ${person.name}\"\"\"" ==>
+ ((DEFAULT_CONTENT_TYPE, 0, 0), (SCALA_MULTI_LINE_STRING, 1, 15), (DEFAULT_CONTENT_TYPE, 16, 28), (SCALA_MULTI_LINE_STRING, 29, 31))
+
+ }
+
}
object ScalaPartitionTokeniserTest {
@@ -85,7 +109,7 @@ object ScalaPartitionTokeniserTest {
class PimpedString(source: String) {
def ==>(expectedPartitions: List[(String, Int, Int)]) {
val actualPartitions = ScalaPartitionTokeniser.tokenise(source)
- assertEquals(expectedPartitions map ScalaPartitionRegion.tupled toList, actualPartitions)
+ assertEquals(source, expectedPartitions map ScalaPartitionRegion.tupled toList, actualPartitions)
}
def ==>(expectedPartitions: (String, Int, Int)*) { this ==> expectedPartitions.toList }
}
View
5 org.scala-ide.sdt.core/src/scala/tools/eclipse/ScalaSourceViewerConfiguration.scala
@@ -36,11 +36,12 @@ import scala.tools.eclipse.formatter.ScalaFormattingStrategy
import scala.tools.eclipse.ui.AutoCloseBracketStrategy
import scala.tools.eclipse.properties.syntaxcolouring.ScalaSyntaxClasses
import scala.tools.eclipse.hyperlink.text.detector.HyperlinksDetector
+import scalariform.ScalaVersions
class ScalaSourceViewerConfiguration(store: IPreferenceStore, scalaPreferenceStore: IPreferenceStore, editor: ITextEditor)
extends JavaSourceViewerConfiguration(JavaPlugin.getDefault.getJavaTextTools.getColorManager, store, editor, IJavaPartitions.JAVA_PARTITIONING) {
- private val codeScanner = new ScalaCodeScanner(getColorManager, store)
+ private val codeScanner = new ScalaCodeScanner(getColorManager, store, ScalaVersions.DEFAULT)
override def getPresentationReconciler(sv: ISourceViewer) = {
val reconciler = super.getPresentationReconciler(sv).asInstanceOf[PresentationReconciler]
@@ -70,7 +71,7 @@ class ScalaSourceViewerConfiguration(store: IPreferenceStore, scalaPreferenceSto
reconciler
}
- private val scalaCodeScanner = new ScalaCodeScanner(getColorManager, scalaPreferenceStore)
+ private val scalaCodeScanner = new ScalaCodeScanner(getColorManager, scalaPreferenceStore, ScalaVersions.DEFAULT)
private val singleLineCommentScanner = new SingleTokenScanner(ScalaSyntaxClasses.SINGLE_LINE_COMMENT, getColorManager, scalaPreferenceStore)
private val multiLineCommentScanner = new SingleTokenScanner(ScalaSyntaxClasses.MULTI_LINE_COMMENT, getColorManager, scalaPreferenceStore)
private val scaladocScanner = new SingleTokenScanner(ScalaSyntaxClasses.SCALADOC, getColorManager, scalaPreferenceStore)
View
76 org.scala-ide.sdt.core/src/scala/tools/eclipse/lexical/ScalaCodeScanner.scala
@@ -1,42 +1,80 @@
package scala.tools.eclipse.lexical
-import org.eclipse.jface.text._
-import org.eclipse.jface.text.rules._
+import scala.annotation.tailrec
+import scala.tools.eclipse.properties.syntaxcolouring._
+
import org.eclipse.jdt.ui.text.IColorManager
import org.eclipse.jface.preference.IPreferenceStore
-import org.eclipse.jdt.ui.PreferenceConstants
-import org.eclipse.jdt.internal.ui.javaeditor.SemanticHighlightings
-import org.eclipse.jdt.internal.ui.text.AbstractJavaScanner
-import scalariform.lexer.{ ScalaLexer, UnicodeEscapeReader, ScalaOnlyLexer }
+import org.eclipse.jface.text.IDocument
+import org.eclipse.jface.text.rules._
+
+import scalariform._
+import scalariform.lexer.ScalaLexer
+import scalariform.lexer.{ Token => ScalariformToken }
import scalariform.lexer.Tokens._
-import scala.tools.eclipse.ScalaPlugin
-import org.eclipse.jface.util.PropertyChangeEvent
-import scala.tools.eclipse.properties.syntaxcolouring.ScalariformToSyntaxClass
-class ScalaCodeScanner(val colorManager: IColorManager, val preferenceStore: IPreferenceStore) extends AbstractScalaScanner {
+class ScalaCodeScanner(val colorManager: IColorManager, val preferenceStore: IPreferenceStore, scalaVersion: ScalaVersion) extends AbstractScalaScanner {
def nextToken(): IToken = {
- val scalaToken = lexer.nextToken()
- getTokenLength = scalaToken.length
- getTokenOffset = scalaToken.offset + offset
- scalaToken.tokenType match {
- case WS => Token.WHITESPACE
- case EOF => Token.EOF
- case _ => getToken(ScalariformToSyntaxClass(scalaToken))
+ val token = tokens(pos)
+
+ getTokenLength = token.length
+ getTokenOffset = token.offset + offset
+
+ val result = token.tokenType match {
+ case WS => Token.WHITESPACE
+ case EOF => Token.EOF
+ case _ if isMacro(token) => getToken(ScalaSyntaxClasses.KEYWORD)
+ case _ => getToken(ScalariformToSyntaxClass(token))
}
+ if (pos + 1 < tokens.length)
+ pos += 1
+ result
}
+ /**
+ * Heuristic to distinguish the macro keyword from uses as an identifier. To be 100% accurate requires a full parse,
+ * which would be too slow, but this is hopefully adequate.
+ */
+ private def isMacro(token: ScalariformToken) =
+ scalaVersion >= ScalaVersions.Scala_2_10 &&
+ token.tokenType.isId && token.text == "macro" &&
+ findMeaningfulToken(pos + 1, shift = 1).exists(token => token.tokenType.isId) &&
+ findMeaningfulToken(pos - 1, shift = -1).exists(_.tokenType == EQUALS)
+
+ /**
+ * Scan forwards or backwards for nearest comment that is neither whitespace nor comment
+ */
+ @tailrec
+ private def findMeaningfulToken(pos: Int, shift: Int): Option[ScalariformToken] =
+ if (pos <= 0 || pos > tokens.length)
+ None
+ else {
+ val token = tokens(pos)
+ token.tokenType match {
+ case WS | LINE_COMMENT | MULTILINE_COMMENT =>
+ findMeaningfulToken(pos + shift, shift)
+ case _ =>
+ Some(token)
+ }
+ }
+
var getTokenOffset: Int = _
var getTokenLength: Int = _
+
+ private var tokens: Array[ScalariformToken] = Array()
+ private var pos = 0
+
private var document: IDocument = _
- private var lexer: ScalaLexer = _
private var offset: Int = _
def setRange(document: IDocument, offset: Int, length: Int) {
this.document = document
this.offset = offset
val source = document.get(offset, length)
- this.lexer = ScalaLexer.createRawLexer(source, forgiveErrors = true)
+ val lexer = ScalaLexer.createRawLexer(source, forgiveErrors = true)
+ this.tokens = lexer.toArray
+ this.pos = 0
}
}
View
108 org.scala-ide.sdt.core/src/scala/tools/eclipse/lexical/ScalaPartitionTokeniser.scala
@@ -65,10 +65,16 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
def nextToken(): ScalaPartitionRegion = {
require(tokensRemain)
- if (modeStack.head.isInstanceOf[ScalaState])
- getScalaToken()
- else
- getXmlToken()
+
+ modeStack.head match {
+ case ScalaState(_) =>
+ getScalaToken()
+ case XmlState(_, _) =>
+ getXmlToken()
+ case StringInterpolationState(multiline, embeddedIdentifierNext) =>
+ getStringInterpolationToken(multiline, embeddedIdentifierNext)
+ }
+
val contentType = contentTypeOpt.get
val tokenStart = previousTokenEnd + 1
val tokenEnd = pos - 1
@@ -90,14 +96,18 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
getOrdinaryScala()
}
case '"' =>
- if (ch(1) == '"' && ch(2) == '"') {
+ val multiline = ch(1) == '"' && ch(2) == '"'
+ val isInterpolation = Character.isUnicodeIdentifierPart(ch(-1)) // TODO: More precise detection
+ if (isInterpolation)
+ nestIntoStringInterpolationMode(multiline)
+ if (multiline) {
setContentType(SCALA_MULTI_LINE_STRING)
accept(3)
- getMultiLineStringLit(quotesRequired = 3)
+ getMultiLineStringLit(quotesRequired = 3, isInterpolation)
} else {
setContentType(JAVA_STRING)
accept()
- getStringLit()
+ getStringLit(isInterpolation)
}
case '/' =>
(ch(1): @switch) match {
@@ -161,18 +171,37 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
None
@tailrec
- private def getStringLit(): Unit =
+ private def getStringLit(isInterpolation: Boolean): Unit =
(ch: @switch) match {
- case '"' => accept()
+ case '"' =>
+ accept()
+ if (isInterpolation)
+ modeStack.pop()
case EOF =>
- case '\n' => accept()
+ if (isInterpolation)
+ modeStack.pop()
+ case '\n' =>
+ accept()
+ if (isInterpolation)
+ modeStack.pop()
case '\r' if ch(1) != '\n' =>
+ if (isInterpolation)
+ modeStack.pop()
case '\\' if ch(1) == '"' || ch(1) == '\\' =>
accept(2)
- getStringLit()
+ getStringLit(isInterpolation)
+ case '$' if ch(1) == '$' =>
+ accept(2)
+ getStringLit(isInterpolation)
+ case '$' if isInterpolation && ch(1) == '{' =>
+ accept()
+ nestIntoScalaMode()
+ case '$' if isInterpolation =>
+ accept()
+ stringInterpolationState.embeddedIdentifierNext = true
case _ =>
accept()
- getStringLit()
+ getStringLit(isInterpolation)
}
@tailrec
@@ -188,17 +217,29 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
}
@tailrec
- private def getMultiLineStringLit(quotesRequired: Int): Unit =
+ private def getMultiLineStringLit(quotesRequired: Int, isInterpolation: Boolean): Unit =
(ch: @switch) match {
case '"' =>
accept()
- getMultiLineStringLit(quotesRequired - 1)
+ getMultiLineStringLit(quotesRequired - 1, isInterpolation)
case EOF =>
+ if (isInterpolation)
+ modeStack.pop()
+ case '$' if ch(1) == '$' =>
+ accept(2)
+ getMultiLineStringLit(quotesRequired, isInterpolation)
+ case '$' if isInterpolation && ch(1) == '{' =>
+ accept()
+ nestIntoScalaMode()
+ case '$' if isInterpolation =>
+ accept()
+ stringInterpolationState.embeddedIdentifierNext = true
case _ =>
if (quotesRequired > 0) {
accept()
- getMultiLineStringLit(3)
- }
+ getMultiLineStringLit(3, isInterpolation)
+ } else if (isInterpolation)
+ modeStack.pop()
}
@tailrec
@@ -265,9 +306,29 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
getSingleLineComment()
}
- private abstract trait ScannerMode
- private class XmlState(var nesting: Int, var inTag: Option[Boolean]) extends ScannerMode
- private class ScalaState(var nesting: Int) extends ScannerMode
+ private def getStringInterpolationToken(multiline: Boolean, embeddedIdentifierNext: Boolean) {
+ if (embeddedIdentifierNext) {
+ setContentType(DEFAULT_CONTENT_TYPE)
+ stringInterpolationState.embeddedIdentifierNext = false
+ do
+ accept()
+ while (ch != EOF && Character.isUnicodeIdentifierPart(ch))
+ } else {
+ if (multiline) {
+ setContentType(SCALA_MULTI_LINE_STRING)
+ getMultiLineStringLit(quotesRequired = 3, isInterpolation = true)
+ } else {
+ setContentType(JAVA_STRING)
+ getStringLit(isInterpolation = true)
+ }
+ }
+ }
+
+ private sealed trait ScannerMode
+ private case class XmlState(var nesting: Int, var inTag: Option[Boolean]) extends ScannerMode
+ private case class ScalaState(var nesting: Int) extends ScannerMode
+ private case class StringInterpolationState(multiline: Boolean, var embeddedIdentifierNext: Boolean) extends ScannerMode
+
private val modeStack: Stack[ScannerMode] = {
val stack = new Stack[ScannerMode]
stack.push(new ScalaState(nesting = 0))
@@ -276,13 +337,18 @@ class ScalaPartitionTokeniser(text: String) extends TokenTests {
private def xmlState = modeStack.head.asInstanceOf[XmlState]
private def scalaState = modeStack.head.asInstanceOf[ScalaState]
+ private def stringInterpolationState = modeStack.head.asInstanceOf[StringInterpolationState]
private def nestIntoScalaMode() {
- modeStack.push(new ScalaState(nesting = 0))
+ modeStack.push(ScalaState(nesting = 0))
}
private def nestIntoXmlMode() {
- modeStack.push(new XmlState(nesting = 0, inTag = None))
+ modeStack.push(XmlState(nesting = 0, inTag = None))
+ }
+
+ private def nestIntoStringInterpolationMode(multiline: Boolean) {
+ modeStack.push(StringInterpolationState(multiline, embeddedIdentifierNext = false))
}
private def getXmlToken(): Unit =
View
2 ...-ide.sdt.core/src/scala/tools/eclipse/semantichighlighting/classifier/SyntacticInfo.scala
@@ -19,7 +19,7 @@ case class SyntacticInfo(
object SyntacticInfo {
private def safeParse(source: String): Option[(CompilationUnit, List[Token])] = {
- val (hiddenTokenInfo, tokens) = ScalaLexer.tokeniseFull(source, forgiveErrors = true)
+ val tokens = ScalaLexer.tokenise(source, forgiveErrors = true)
val parser = new ScalaParser(tokens.toArray)
parser.safeParse(parser.compilationUnitOrScript) map { (_, tokens) }
}
View
2 org.scala-ide.sdt.core/src/scala/tools/eclipse/semicolon/InferredSemicolonPainter.scala
@@ -99,7 +99,7 @@ class InferredSemicolonPainter(textViewer: ITextViewer with ITextViewerExtension
private def findInferredSemis: List[Token] =
try {
- val (hiddenTokenInfo, tokens) = ScalaLexer.tokeniseFull(textViewer.getDocument.get)
+ val tokens = ScalaLexer.tokenise(textViewer.getDocument.get)
InferredSemicolonScalaParser.findSemicolons(tokens.toArray).toList
} catch {
case e: ScalaParserException => Nil

0 comments on commit 7e1e012

Please sign in to comment.
Something went wrong with that request. Please try again.