-
-
Notifications
You must be signed in to change notification settings - Fork 242
/
NetLogoTokenMaker.scala
133 lines (108 loc) · 4.97 KB
/
NetLogoTokenMaker.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
// (C) Uri Wilensky. https://github.com/NetLogo/NetLogo
package org.nlogo.ide
import javax.swing.text.Segment
import org.fife.ui.rsyntaxtextarea.{ TokenMakerBase }
import org.fife.ui.rsyntaxtextarea.{ Token => RstaToken, TokenImpl, TokenTypes }
import org.nlogo.api.{ NetLogoLegacyDialect, NetLogoThreeDDialect }
import org.nlogo.core.{ Dialect, Femto, Nobody, Token, TokenizerInterface, TokenType }
import org.nlogo.nvm.ExtensionManager
trait NetLogoTokenMaker extends TokenMakerBase {
def extensionManager: Option[ExtensionManager]
def dialect: Dialect
val tokenizer = Femto.scalaSingleton[TokenizerInterface]("org.nlogo.lex.Tokenizer")
val namer = Femto.scalaSingleton[Token => Token]("org.nlogo.parse.Namer0")
def literalToken(value: AnyRef): Int =
value match {
case s: String => TokenTypes.LITERAL_STRING_DOUBLE_QUOTE
case d: java.lang.Double => TokenTypes.LITERAL_NUMBER_FLOAT
case b: java.lang.Boolean => TokenTypes.LITERAL_BOOLEAN
case Nobody => TokenTypes.LITERAL_BACKQUOTE
case _ => TokenTypes.IDENTIFIER
}
def rstaTokenType(t: Token, firstOnLine: Boolean): Int = {
import TokenType._
def isBreedVariable = {
(dialect.agentVariables.implicitObserverVariableTypeMap.keySet ++
dialect.agentVariables.implicitTurtleVariableTypeMap.keySet ++
dialect.agentVariables.implicitPatchVariableTypeMap.keySet ++
dialect.agentVariables.implicitLinkVariableTypeMap.keySet).contains(t.text.toUpperCase)
}
def tagToken(t: Token): Token = {
val named = namer(t)
named.tpe match {
case TokenType.Ident =>
extensionManager.flatMap { manager =>
manager.cachedType(t.text.toUpperCase)
.map(tokenType => t.copy(tpe = tokenType))
}.getOrElse(t)
case _ => named
}
}
val punctType = TokenTypes.SEPARATOR
t.tpe match {
case OpenParen | CloseParen | OpenBracket | CloseBracket | OpenBrace | CloseBrace | Comma => punctType
case Literal => literalToken(t.value)
case Ident =>
val namedToken = tagToken(t)
namedToken.tpe match {
case Command => TokenTypes.OPERATOR
case Reporter => TokenTypes.FUNCTION
case Keyword => TokenTypes.RESERVED_WORD
case Literal => literalToken(namedToken.value)
case _ =>
if (dialect.tokenMapper.getCommand(t.text.toUpperCase).isDefined)
TokenTypes.OPERATOR
else if (dialect.tokenMapper.getReporter(t.text.toUpperCase).isDefined)
TokenTypes.FUNCTION
else if (t.text.equalsIgnoreCase("BREED") && firstOnLine)
TokenTypes.RESERVED_WORD
else if (isBreedVariable)
TokenTypes.FUNCTION
else
TokenTypes.IDENTIFIER
}
case Command => TokenTypes.OPERATOR
case Reporter => TokenTypes.FUNCTION
case Keyword => TokenTypes.RESERVED_WORD
case Comment => TokenTypes.COMMENT_KEYWORD
case Bad => TokenTypes.ERROR_IDENTIFIER
case Extension => TokenTypes.IDENTIFIER
case Whitespace => TokenTypes.WHITESPACE
case Eof => TokenTypes.NULL
}
}
// trying to use `resetTokenList()` directly causes the following scala compiler error:
// "Implementation restriction: trait NetLogoTokenMaker accesses protected method resetTokenList inside a concrete trait method."
// Adding this proxy method to each class extending class TokenMakerBase works around the issue.
def _resetTokenList(): Unit
def getTokenList(seg: Segment, initialTokenType: Int, offset: Int): RstaToken = {
_resetTokenList()
val offsetShift = - seg.offset + offset
seg.setIndex(seg.getBeginIndex) // reset Segment
def netlogoTokenToRstaToken(netLogoToken: Token, lastToken: Option[TokenImpl]): TokenImpl = {
val next = new TokenImpl(seg, netLogoToken.start, netLogoToken.end - 1, offsetShift + netLogoToken.start, rstaTokenType(netLogoToken, lastToken.isEmpty), 0)
lastToken.foreach { last =>
last.setNextToken(next)
}
next
}
val tokens = tokenizer.tokenizeWithWhitespace(seg, "").filter(_.tpe != TokenType.Eof)
val firstRstaToken =
if (tokens.hasNext) netlogoTokenToRstaToken(tokens.next(), None)
else new TokenImpl(seg, seg.getIndex, seg.getIndex, offset, TokenTypes.NULL, 0)
tokens.foldLeft(firstRstaToken) {
case (rstaToken, nlToken) => netlogoTokenToRstaToken(nlToken, Some(rstaToken))
}
firstRstaToken
}
}
class NetLogoTwoDTokenMaker(val extensionManager: Option[ExtensionManager]) extends NetLogoTokenMaker {
def this() = this(None)
def dialect = NetLogoLegacyDialect
def _resetTokenList(): Unit = { super.resetTokenList() }
}
class NetLogoThreeDTokenMaker(val extensionManager: Option[ExtensionManager]) extends NetLogoTokenMaker {
def this() = this(None)
def dialect = NetLogoThreeDDialect
def _resetTokenList(): Unit = { super.resetTokenList() }
}