Skip to content
Browse files

Make TokenTypes for all reserved words.

  • Loading branch information...
1 parent 9de20ae commit 17aa4238c46024d8cc77d785da64a726d8c58c68 @munificent committed Feb 25, 2012
View
2 lib/magpie/syntax.mag
@@ -1,5 +1,3 @@
-import classfile:com.stuffwithstuff.magpie.intrinsic.SyntaxMethods
-
// TODO(bob): Need to make sure this cannot be constructed somehow.
defclass Parser
/// An instance of this class will be passed to a user-defined parse()
View
6 src/com/stuffwithstuff/magpie/app/NiceReplCharacterReader.java
@@ -58,19 +58,21 @@ protected void afterReadLine(Repl repl, String prompt, String line) {
// Identifiers.
case NAME:
+ /*
if (token.isKeyword("this") ||
token.isKeyword("nothing") ||
token.isKeyword("it")) {
// special identifiers
Term.set(ForeColor.LIGHT_BLUE);
// TODO(bob): Fix this once we have token types for the reserved words.
- /*
} else if (repl.isKeyword(token.getString())) {
Term.set(ForeColor.CYAN);
- */
} else {
+ */
Term.set(ForeColor.WHITE);
+ /*
}
+ */
break;
case FIELD:
View
189 src/com/stuffwithstuff/magpie/intrinsic/SyntaxMethods.java
@@ -1,189 +0,0 @@
-package com.stuffwithstuff.magpie.intrinsic;
-
-/*
-import java.util.List;
-
-import com.stuffwithstuff.magpie.Def;
-import com.stuffwithstuff.magpie.ast.Expr;
-import com.stuffwithstuff.magpie.interpreter.ClassObj;
-import com.stuffwithstuff.magpie.interpreter.Context;
-import com.stuffwithstuff.magpie.interpreter.JavaToMagpie;
-import com.stuffwithstuff.magpie.interpreter.MagpieToJava;
-import com.stuffwithstuff.magpie.interpreter.Obj;
-import com.stuffwithstuff.magpie.parser.InfixParser;
-import com.stuffwithstuff.magpie.parser.MagpieParser;
-import com.stuffwithstuff.magpie.parser.PrefixParser;
-import com.stuffwithstuff.magpie.parser.Token;
-import com.stuffwithstuff.magpie.parser.TokenType;
-import com.stuffwithstuff.magpie.util.Pair;
-*/
-
-// TODO(bob): Now that Magpie's grammar isn't extensible, this stuff isn't
-// being used. Keeping it around though because it may still be useful as a
-// module for playing with Magpie code in Magpie. When Magpie is self-hosted,
-// this can go away entirely.
-
-public class SyntaxMethods {
- /*
- @Def("definePrefix(keyword is String, parser)")
- public static class DefinePrefix implements Intrinsic {
- public Obj invoke(Context context, Obj left, Obj right) {
- String keyword = right.getField(0).asString();
- Obj parser = right.getField(1);
-
- context.getModule().defineSyntax(keyword,
- new MagpiePrefixParser(context, parser), true);
-
- return context.nothing();
- }
- }
-
- @Def("defineInfix(keyword is String, precedence is Int, parser)")
- public static class DefineInfix implements Intrinsic {
- public Obj invoke(Context context, Obj left, Obj right) {
- String keyword = right.getField(0).asString();
- int precedence = right.getField(1).asInt();
- Obj parser = right.getField(2);
-
- context.getModule().defineSyntax(keyword,
- new MagpieInfixParser(context, precedence, parser), true);
-
- return context.nothing();
- }
- }
-
- @Def("(this is Parser) consume(keyword is String)")
- public static class Consume_Keyword implements Intrinsic {
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
- String keyword = right.asString();
-
- return JavaToMagpie.convert(context, parser.consume(keyword));
- }
- }
-
- @Def("(this is Parser) matchToken(token is String)")
- public static class MatchToken_String implements Intrinsic {
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
- String keyword = right.asString();
-
- return context.toObj(parser.match(keyword));
- }
- }
-
- @Def("(this is Parser) matchToken(token is TokenType)")
- public static class MatchToken_TokenType implements Intrinsic {
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
-
- TokenType tokenType = MagpieToJava.convertTokenType(
- context, right);
- return context.toObj(parser.match(tokenType));
- }
- }
-
- @Def("(this is Parser) parseExpression(precedence is Int)")
- public static class ParseExpression implements Intrinsic {
- @Override
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
-
- Expr expr = parser.parsePrecedence(right.asInt());
-
- return JavaToMagpie.convert(context, expr);
- }
- }
-
- @Def("(this is Parser) parseExpressionOrBlock(keywords is Array)")
- public static class ParseExpressionOrBlock_List implements Intrinsic {
- @Override
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
-
- List<Obj> keywordObjs = right.asList();
- String[] keywords = new String[keywordObjs.size()];
- for (int i = 0; i < keywords.length; i++) {
- keywords[i] = keywordObjs.get(i).asString();
- }
-
- Pair<Expr, Token> result = parser.parseExpressionOrBlock(keywords);
-
- Obj expr = JavaToMagpie.convert(context, result.getKey());
- Obj token = JavaToMagpie.convert(context, result.getValue());
- return context.toObj(expr, token);
- }
- }
-
- @Def("(this is Parser) parseExpressionOrBlock()")
- public static class ParseExpressionOrBlock_Nothing implements Intrinsic {
- @Override
- public Obj invoke(Context context, Obj left, Obj right) {
- MagpieParser parser = (MagpieParser) left.getValue();
-
- Expr expr = parser.parseExpressionOrBlock();
- return JavaToMagpie.convert(context, expr);
- }
- }
-
- private static class MagpiePrefixParser implements PrefixParser {
- public MagpiePrefixParser(Context context, Obj parser) {
- mContext = context;
- mParser = parser;
- }
-
- @Override
- public Expr parse(MagpieParser parser, Token token) {
- // Wrap the Java parser in a Magpie one.
- ClassObj parserClass = mContext.getInterpreter().getSyntaxModule().getScope().get("Parser").asClass();
- Obj parserObj = mContext.instantiate(parserClass, parser);
-
- Obj tokenObj = JavaToMagpie.convert(mContext, token);
- Obj arg = mContext.toObj(parserObj, tokenObj);
-
- // Let the Magpie code do the parsing. Call the parser.
- Obj expr = mContext.getInterpreter().invoke(mParser, "call", arg);
-
- // Marshall it back to Java format.
- return MagpieToJava.convertExpr(mContext, expr);
- }
-
- private final Context mContext;
- private final Obj mParser;
- }
-
- private static class MagpieInfixParser implements InfixParser {
- public MagpieInfixParser(Context context, int precedence, Obj parser) {
- mContext = context;
- mPrecedence = precedence;
- mParser = parser;
- }
-
- @Override
- public Expr parse(MagpieParser parser, Expr left, Token token) {
- // Wrap the Java parser in a Magpie one.
- ClassObj parserClass = mContext.getInterpreter().getSyntaxModule().getScope().get("Parser").asClass();
- Obj parserObj = mContext.instantiate(parserClass, parser);
-
- Obj tokenObj = JavaToMagpie.convert(mContext, token);
- Obj leftObj = JavaToMagpie.convert(mContext, left);
- Obj arg = mContext.toObj(parserObj, leftObj, tokenObj);
-
- // Let the Magpie code do the parsing. Call the parser.
- Obj expr = mContext.getInterpreter().invoke(mParser, "call", arg);
-
- // Marshall it back to Java format.
- return MagpieToJava.convertExpr(mContext, expr);
- }
-
- @Override
- public int getPrecedence() {
- return mPrecedence;
- }
-
- private final Context mContext;
- private final int mPrecedence;
- private final Obj mParser;
- }
- */
-}
View
5 src/com/stuffwithstuff/magpie/parser/Annotator.java
@@ -36,9 +36,4 @@ public Token readToken() {
return token;
}
}
-
- @Override
- protected boolean isReserved(String name) {
- return false;
- }
}
View
68 src/com/stuffwithstuff/magpie/parser/Grammar.java
@@ -1,8 +1,5 @@
package com.stuffwithstuff.magpie.parser;
-import java.util.HashSet;
-import java.util.Set;
-
public class Grammar {
public Grammar() {
prefix(TokenType.BOOL, new LiteralParser());
@@ -16,7 +13,7 @@ public Grammar() {
prefix(TokenType.NAME, new NameParser());
prefix(TokenType.FIELD, new FieldParser());
- prefix("fn", new FnParser());
+ prefix(TokenType.FN, new FnParser());
infix(TokenType.AND, new AndParser());
infix(TokenType.OR, new OrParser());
@@ -36,9 +33,6 @@ public Grammar() {
infix(TokenType.GTE, new InfixOperatorParser(5));
infix(TokenType.EQEQ, new InfixOperatorParser(4));
infix(TokenType.NOTEQ, new InfixOperatorParser(4));
-
- reserve("break case catch def defclass do else end export for");
- reserve("import if in is match return then throw val var while");
}
public PrefixParser getPrefixParser(Token token) {
@@ -57,55 +51,8 @@ public InfixParser getInfixParser(String keyword) {
return mInfixParsers.get(keyword);
}
- public void defineParser(String keyword, PrefixParser parser) {
- mPrefixParsers.define(keyword, parser);
- }
-
- public void defineParser(String keyword, InfixParser parser) {
- mInfixParsers.define(keyword, parser);
- }
-
- public boolean isKeyword(String name) {
- return mReservedWords.contains(name) ||
- mPrefixParsers.contains(name) ||
- mInfixParsers.contains(name);
- }
- /**
- * Gets whether or not this token is a reserved word. Reserved words like
- * "else" and "then" are claimed for special use by mixfix parsers, so can't
- * be parsed on their own.
- *
- * @param token The token to test
- * @return True if the token is a reserved name token.
- */
- public boolean isReserved(String name) {
- return mReservedWords.contains(name);
- }
-
- /**
- * Gets whether or not this token is a reserved word. Reserved words like
- * "else" and "then" are claimed for special use by mixfix parsers, so can't
- * be parsed on their own.
- *
- * @param token The token to test
- * @return True if the token is a reserved name token.
- */
- public boolean isReserved(Token token) {
- return (token.getType() == TokenType.NAME) &&
- isReserved(token.getString());
- }
-
public int getPrecedence(Token token) {
int precedence = 0;
-
- // A reserved word can't start an infix expression. Prevents us from
- // parsing a keyword as an identifier.
- if (isReserved(token)) return 0;
-
- // If we have a prefix parser for this token's name, then that takes
- // precedence. Prevents us from parsing a keyword as an identifier.
- if ((token.getValue() instanceof String) &&
- mPrefixParsers.isReserved(token.getString())) return 0;
InfixParser parser = mInfixParsers.get(token);
if (parser != null) {
@@ -119,25 +66,12 @@ private void prefix(TokenType type, PrefixParser parser) {
mPrefixParsers.define(type, parser);
}
- private void prefix(String keyword, PrefixParser parser) {
- mReservedWords.add(keyword);
- mPrefixParsers.define(keyword, parser);
- }
-
private void infix(TokenType type, InfixParser parser) {
mInfixParsers.define(type, parser);
}
- private void reserve(String wordString) {
- String[] words = wordString.split(" ");
- for (int i = 0; i < words.length; i++) {
- mReservedWords.add(words[i]);
- }
- }
-
private final ParserTable<PrefixParser> mPrefixParsers =
new ParserTable<PrefixParser>();
private final ParserTable<InfixParser> mInfixParsers =
new ParserTable<InfixParser>();
- private final Set<String> mReservedWords = new HashSet<String>();
}
View
82 src/com/stuffwithstuff/magpie/parser/Lexer.java
@@ -1,5 +1,8 @@
package com.stuffwithstuff.magpie.parser;
+import java.util.HashMap;
+import java.util.Map;
+
import com.stuffwithstuff.magpie.SourceReader;
import com.stuffwithstuff.magpie.parser.Token;
import com.stuffwithstuff.magpie.parser.TokenType;
@@ -282,42 +285,14 @@ private Token makeToken(TokenType type) {
private Token makeToken(TokenType type, Object value) {
// Handle reserved words.
if (type == TokenType.NAME) {
- if (mRead.equals("and")) {
- type = TokenType.AND;
- } else if (mRead.equals("or")) {
- type = TokenType.OR;
- } else if (mRead.equals("nothing")) {
- type = TokenType.NOTHING;
+ if (sKeywords.containsKey(mRead)) {
+ type = sKeywords.get(mRead);
} else if (mRead.equals("false")) {
type = TokenType.BOOL;
value = false;
} else if (mRead.equals("true")) {
type = TokenType.BOOL;
value = true;
- } else if (mRead.equals("*")) {
- type = TokenType.ASTERISK;
- } else if (mRead.equals("/")) {
- type = TokenType.SLASH;
- } else if (mRead.equals("%")) {
- type = TokenType.PERCENT;
- } else if (mRead.equals("+")) {
- type = TokenType.PLUS;
- } else if (mRead.equals("-")) {
- type = TokenType.MINUS;
- } else if (mRead.equals("<")) {
- type = TokenType.LT;
- } else if (mRead.equals(">")) {
- type = TokenType.GT;
- } else if (mRead.equals("<=")) {
- type = TokenType.LTE;
- } else if (mRead.equals(">=")) {
- type = TokenType.GTE;
- } else if (mRead.equals("=")) {
- type = TokenType.EQ;
- } else if (mRead.equals("==")) {
- type = TokenType.EQEQ;
- } else if (mRead.equals("!=")) {
- type = TokenType.NOTEQ;
}
}
@@ -355,4 +330,51 @@ private boolean isOperator(final char c) {
private int mStartCol;
private int mLine;
private int mCol;
+
+ private static final Map<String, TokenType> sKeywords;
+
+ static {
+ sKeywords = new HashMap<String, TokenType>();
+ sKeywords.put("and", TokenType.AND);
+ sKeywords.put("as", TokenType.AS);
+ sKeywords.put("break", TokenType.BREAK);
+ sKeywords.put("case", TokenType.CASE);
+ sKeywords.put("catch", TokenType.CATCH);
+ sKeywords.put("def", TokenType.DEF);
+ sKeywords.put("defclass", TokenType.DEFCLASS);
+ sKeywords.put("do", TokenType.DO);
+ sKeywords.put("else", TokenType.ELSE);
+ sKeywords.put("end", TokenType.END);
+ sKeywords.put("export", TokenType.EXPORT);
+ sKeywords.put("fn", TokenType.FN);
+ sKeywords.put("for", TokenType.FOR);
+ sKeywords.put("import", TokenType.IMPORT);
+ sKeywords.put("if", TokenType.IF);
+ sKeywords.put("in", TokenType.IN);
+ sKeywords.put("is", TokenType.IS);
+ sKeywords.put("match", TokenType.MATCH);
+ sKeywords.put("nothing", TokenType.NOTHING);
+ sKeywords.put("only", TokenType.ONLY);
+ sKeywords.put("or", TokenType.OR);
+ sKeywords.put("return", TokenType.RETURN);
+ sKeywords.put("then", TokenType.THEN);
+ sKeywords.put("throw", TokenType.THROW);
+ sKeywords.put("val", TokenType.VAL);
+ sKeywords.put("var", TokenType.VAR);
+ sKeywords.put("while", TokenType.WHILE);
+ sKeywords.put("with", TokenType.WITH);
+
+ sKeywords.put("*", TokenType.ASTERISK);
+ sKeywords.put("/", TokenType.SLASH);
+ sKeywords.put("%", TokenType.PERCENT);
+ sKeywords.put("+", TokenType.PLUS);
+ sKeywords.put("-", TokenType.MINUS);
+ sKeywords.put("<", TokenType.LT);
+ sKeywords.put(">", TokenType.GT);
+ sKeywords.put("<=", TokenType.LTE);
+ sKeywords.put(">=", TokenType.GTE);
+ sKeywords.put("=", TokenType.EQ);
+ sKeywords.put("==", TokenType.EQEQ);
+ sKeywords.put("!=", TokenType.NOTEQ);
+ }
}
View
110 src/com/stuffwithstuff/magpie/parser/MagpieParser.java
@@ -47,19 +47,19 @@ public MagpieParser(TokenReader tokens) {
* the condition in an "if" expection.
*/
public Expr parseStatement() {
- if (match("break")) return parseBreak();
- if (match("def")) return parseDef();
- if (match("defclass")) return parseDefclass();
- if (match("do")) return parseDo();
- if (match("for")) return parseLoop();
- if (match("if")) return parseIf();
- if (match("import")) return parseImport();
- if (match("match")) return parseMatch();
- if (match("return")) return parseReturn();
- if (match("throw")) return parseThrow();
- if (match("var")) return parseVar(true);
- if (match("val")) return parseVar(false);
- if (match("while")) return parseLoop();
+ if (match(TokenType.BREAK)) return parseBreak();
+ if (match(TokenType.DEF)) return parseDef();
+ if (match(TokenType.DEFCLASS)) return parseDefclass();
+ if (match(TokenType.DO)) return parseDo();
+ if (match(TokenType.FOR)) return parseLoop();
+ if (match(TokenType.IF)) return parseIf();
+ if (match(TokenType.IMPORT)) return parseImport();
+ if (match(TokenType.MATCH)) return parseMatch();
+ if (match(TokenType.RETURN)) return parseReturn();
+ if (match(TokenType.THROW)) return parseThrow();
+ if (match(TokenType.VAR)) return parseVar(true);
+ if (match(TokenType.VAL)) return parseVar(false);
+ if (match(TokenType.WHILE)) return parseLoop();
return parseExpression();
}
@@ -96,14 +96,14 @@ private Expr parseInfix(Expr left, int precedence) {
}
Expr parseBlock() {
- return parseBlock(true, new String[] { "end" }).getKey();
+ return parseBlock(true, new TokenType[] { TokenType.END }).getKey();
}
Expr parseExpressionOrBlock() {
- return parseExpressionOrBlock("end").getKey();
+ return parseExpressionOrBlock(TokenType.END).getKey();
}
- private Pair<Expr, Token> parseExpressionOrBlock(String... endTokens) {
+ private Pair<Expr, Token> parseExpressionOrBlock(TokenType... endTokens) {
return parseExpressionOrBlock(true, endTokens);
}
@@ -239,11 +239,6 @@ public void popQuote() {
mQuoteDepth--;
}
- @Override
- protected boolean isReserved(String name) {
- return mGrammar.isReserved(name);
- }
-
private Expr parseBreak() {
return Expr.break_(last(1).getPosition());
}
@@ -259,7 +254,7 @@ private Expr allowExpressionAfterBlock(Expr expr) {
// Need a more elegant way to handle this.
// Only if we have a block body. Single-expression bodies shouldn't do this.
- if (!last(1).isKeyword("end")) return expr;
+ if (last(1).getType() != TokenType.END) return expr;
return parseInfix(expr, 0);
}
@@ -275,7 +270,7 @@ private Expr parseDef() {
if (match(TokenType.LINE, TokenType.DOC_COMMENT)) {
doc = last(1).getString();
consume(TokenType.LINE);
- consume("end");
+ consume(TokenType.END);
}
return Expr.method(span.end(), doc, name);
@@ -315,7 +310,7 @@ private Expr parseDefclass() {
// Parse the parents, if any.
List<String> parents = new ArrayList<String>();
- if (match("is")) {
+ if (match(TokenType.IS)) {
do {
parents.add(consume(TokenType.NAME).getString());
} while (match(TokenType.COMMA));
@@ -332,9 +327,9 @@ private Expr parseDefclass() {
Map<String, Field> fields = new HashMap<String, Field>();
// Parse the body.
- while (!match("end")) {
- if (match("var")) parseField(true, fields);
- else if (match("val")) parseField(false, fields);
+ while (!match(TokenType.END)) {
+ if (match(TokenType.VAR)) parseField(true, fields);
+ else if (match(TokenType.VAL)) parseField(false, fields);
consume(TokenType.LINE);
}
@@ -382,27 +377,27 @@ private Expr parseImport() {
// Parse the prefix, if any.
String prefix = null;
- if (match("as")) {
+ if (match(TokenType.AS)) {
prefix = consume(TokenType.NAME).getString();
}
// Parse the declarations, if any.
List<ImportDeclaration> declarations = new ArrayList<ImportDeclaration>();
boolean isOnly = false;
- if (match("with")) {
- if (match("only")) isOnly = true;
+ if (match(TokenType.WITH)) {
+ if (match(TokenType.ONLY)) isOnly = true;
consume(TokenType.LINE);
- while (!match("end")) {
+ while (!match(TokenType.END)) {
// TODO(bob): "excluding".
- boolean export = match("export");
+ boolean export = match(TokenType.EXPORT);
String name = consume(TokenType.NAME).getString();
String rename = null;
- if (match("as")) {
+ if (match(TokenType.AS)) {
rename = consume(TokenType.NAME).getString();
}
@@ -424,33 +419,34 @@ private Expr parseMatch() {
// Parse the cases.
List<MatchCase> cases = new ArrayList<MatchCase>();
- while (match("case")) {
+ while (match(TokenType.CASE)) {
cases.add(parseCase());
}
// Parse the else case, if present.
- if (match("else")) {
+ if (match(TokenType.ELSE)) {
Expr elseCase = parseExpressionOrBlock();
cases.add(new MatchCase(elseCase));
}
consume(TokenType.LINE);
- consume("end");
+ consume(TokenType.END);
return allowExpressionAfterBlock(Expr.match(span.end(), value, cases));
}
private MatchCase parseCase() {
Pattern pattern = PatternParser.parse(this);
- consume("then");
+ consume(TokenType.THEN);
- Pair<Expr, Token> bodyParse = parseExpressionOrBlock("else", "end", "case");
+ Pair<Expr, Token> bodyParse = parseExpressionOrBlock(
+ TokenType.ELSE, TokenType.END, TokenType.CASE);
// Allow newlines to separate single-line case and else cases.
if ((bodyParse.getValue() == null) &&
- (lookAhead(TokenType.LINE, "case") ||
- lookAhead(TokenType.LINE, "else"))) {
+ (lookAhead(TokenType.LINE, TokenType.CASE) ||
+ lookAhead(TokenType.LINE, TokenType.ELSE))) {
consume(TokenType.LINE);
}
@@ -495,15 +491,15 @@ private Expr parseLoop() {
List<Expr> eachLoop = new ArrayList<Expr>();
while (true) {
- if (token.isKeyword("while")) {
+ if (token.getType() == TokenType.WHILE) {
Expr condition = parseExpression();
eachLoop.add(Expr.if_(condition,
Expr.nothing(),
Expr.break_(condition.getPosition())));
} else {
PositionSpan iteratorSpan = span();
Pattern pattern = PatternParser.parse(this);
- consume("in");
+ consume(TokenType.IN);
Expr generator = parseExpression();
Position position = iteratorSpan.end();
@@ -525,14 +521,14 @@ private Expr parseLoop() {
}
match(TokenType.LINE); // Optional line after a clause.
- if (match("while") || match("for")) {
+ if (matchAny(TokenType.WHILE, TokenType.FOR)) {
token = last(1);
} else {
break;
}
}
- consume("do");
+ consume(TokenType.DO);
Expr body = parseExpressionOrBlock();
// Build the loop body.
@@ -558,22 +554,22 @@ private Expr parseIf() {
PositionSpan span = span();
// Parse the condition.
- Expr condition = parseExpressionOrBlock(new String[] { "then" }).getKey();
+ Expr condition = parseExpressionOrBlock(TokenType.THEN).getKey();
// Parse the then body.
- consume("then");
+ consume(TokenType.THEN);
Pair<Expr, Token> thenResult = parseExpressionOrBlock(
- new String[] { "else", "end" });
+ TokenType.ELSE, TokenType.END);
Expr thenExpr = thenResult.getKey();
Token endToken = thenResult.getValue();
// Don't try to parse "else" if we got an explicit "end" for the "then"
// block.
- boolean consumedEnd = (endToken != null) && endToken.isKeyword("end");
+ boolean consumedEnd = (endToken != null) && (endToken.getType() == TokenType.END);
// See if we have an "else" keyword and parse the else arm.
Expr elseExpr;
- if (!consumedEnd && match("else")) {
+ if (!consumedEnd && match(TokenType.ELSE)) {
elseExpr = parseExpressionOrBlock();
} else {
elseExpr = Expr.nothing();
@@ -621,7 +617,7 @@ private Expr parseVar(boolean isMutable) {
}
private Pair<Expr, Token> parseExpressionOrBlock(boolean parseCatch,
- Object[] endTokens) {
+ TokenType[] endTokens) {
if (lookAhead(TokenType.LINE)){
return parseBlock(parseCatch, endTokens);
} else {
@@ -631,14 +627,14 @@ private Expr parseVar(boolean isMutable) {
}
private Pair<Expr, Token> parseBlock(boolean parseCatch,
- Object[] endTokens) {
+ TokenType[] endTokens) {
consume(TokenType.LINE);
List<Expr> exprs = new ArrayList<Expr>();
while (true) {
if ((endTokens != null) && lookAheadAny(endTokens)) break;
- if (lookAhead("catch")) break;
+ if (lookAhead(TokenType.CATCH)) break;
exprs.add(parseStatement());
consume(TokenType.LINE);
@@ -649,14 +645,14 @@ private Expr parseVar(boolean isMutable) {
// If the block ends with 'end', then we want to consume that token,
// otherwise we want to leave it unconsumed to be consistent with the
// single-expression block case.
- if (endToken.isKeyword("end")) {
+ if (endToken.getType() == TokenType.END) {
consume();
}
// Parse any catch clauses.
List<MatchCase> catches = new ArrayList<MatchCase>();
if (parseCatch) {
- while (match("catch")) {
+ while (match(TokenType.CATCH)) {
catches.add(parseCatch(endTokens));
}
}
@@ -669,15 +665,15 @@ private Expr parseVar(boolean isMutable) {
return new Pair<Expr, Token>(expr, endToken);
}
- private MatchCase parseCatch(Object[] endTokens) {
+ private MatchCase parseCatch(TokenType[] endTokens) {
Pattern pattern = PatternParser.parse(this);
- consume("then");
+ consume(TokenType.THEN);
Pair<Expr, Token> body = parseExpressionOrBlock(false, endTokens);
// Allow newlines to separate single-line catches.
- if ((body.getValue() == null) && lookAhead(TokenType.LINE, "catch")) {
+ if ((body.getValue() == null) && lookAhead(TokenType.LINE, TokenType.CATCH)) {
consume();
}
View
2 src/com/stuffwithstuff/magpie/parser/NameParser.java
@@ -25,7 +25,7 @@ public Expr parse(MagpieParser parser, Expr left, Token token) {
}
// Pass the block, if any.
- if (parser.match("with")) {
+ if (parser.match(TokenType.WITH)) {
// Parse the parameter list if given.
Pattern blockType;
if (parser.lookAhead(TokenType.LEFT_PAREN)) {
View
78 src/com/stuffwithstuff/magpie/parser/Parser.java
@@ -1,6 +1,7 @@
package com.stuffwithstuff.magpie.parser;
-import java.util.*;
+import java.util.LinkedList;
+import java.util.List;
import com.stuffwithstuff.magpie.util.Expect;
@@ -51,33 +52,16 @@ public Token current() {
/**
* Looks ahead at the token stream to see if the next tokens match the
- * expected ones, in order. For example, if the next tokens are (123, "do"),
- * then a call to lookAhead(TokenType.INT, "do") will return true.
+ * expected types, in order. For example, if the next tokens are (123, "do"),
+ * then a call to lookAhead(TokenType.INT, TokenType.DO) will return true.
* Does not consume any tokens.
*
- * @param tokens The expected tokens. Each element can either be a
- * TokenType or a string denoting the name of a specific
- * keyword.
+ * @param tokens The expected types.
* @return True if the next tokens match.
*/
- public boolean lookAhead(Object... tokens) {
+ public boolean lookAhead(TokenType... tokens) {
for (int i = 0; i < tokens.length; i++) {
- // See if we're matching by type or keyword.
- if (tokens[i] instanceof TokenType) {
- TokenType type = (TokenType)tokens[i];
- if (!lookAhead(i).getType().equals(type)) return false;
-
- // If we're looking for a NAME, we need to make sure that name is not a
- // reserved word.
- if ((type == TokenType.NAME) && isReserved(lookAhead(i).getString())) {
- return false;
- }
- } else {
- // Must be a keyword.
- String keyword = (String)tokens[i];
- if (lookAhead(i).getType() != TokenType.NAME) return false;
- if (!lookAhead(i).getString().equals(keyword)) return false;
- }
+ if (!lookAhead(i).getType().equals(tokens[i])) return false;
}
return true;
@@ -87,11 +71,11 @@ public boolean lookAhead(Object... tokens) {
* Gets whether or not the next Token is of any of the given types. Does not
* consume the token.
*
- * @param tokens The allowed tokens for the next Token.
- * @return True if the Token is one of the tokens.
+ * @param tokens The allowed types for the next Token.
+ * @return True if the Token is one of the types.
*/
- public boolean lookAheadAny(Object... tokens) {
- for (Object token : tokens) {
+ public boolean lookAheadAny(TokenType... tokens) {
+ for (TokenType token : tokens) {
if (lookAhead(token)) return true;
}
@@ -100,13 +84,13 @@ public boolean lookAheadAny(Object... tokens) {
/**
* Looks ahead at the token stream to see if the next tokens match the
- * expected ones, in order. If so, they are all consumed.
+ * expected types, in order. If so, they are all consumed.
*
- * @param tokens The expected tokens, in the order that they are
+ * @param tokens The expected types, in the order that they are
* expected to appear.
* @return True if the tokens matched and were consumed.
*/
- public boolean match(Object... tokens) {
+ public boolean match(TokenType... tokens) {
// See if they match.
if (!lookAhead(tokens)) return false;
@@ -120,13 +104,13 @@ public boolean match(Object... tokens) {
/**
* Looks ahead at the next token to see if it's any of the given allowed
- * tokens. If so, consumes it.
+ * types. If so, consumes it.
*
- * @param tokens The tokens the next token can be.
+ * @param tokens The types the next token can be.
* @return True if it matched and was consumed.
*/
- public boolean matchAny(Object... tokens) {
- for (Object token : tokens) {
+ public boolean matchAny(TokenType... tokens) {
+ for (TokenType token : tokens) {
if (match(token)) return true;
}
@@ -164,31 +148,6 @@ public Token consume(TokenType type) {
}
}
- /**
- * Consumes the current token and advances to the next one. The token is
- * required to be the given keyword. If not, a ParseException will be thrown.
- *
- * @param keyword The keyword that the current token must be.
- * @return The consumed token.
- */
- public Token consume(String keyword) {
- if (match(keyword)) {
- return last(1);
- } else {
- Token current = current();
- String message = String.format("Expected keyword %s, found %s.",
- keyword, current);
- throw new ParseException(current.getPosition(), message);
- }
- }
-
- /**
- * Gets whether or not the name is reserved. Reserved words describe name
- * tokens whose name is special and prohibit the token from being parsed as
- * a regular identifier. For example, "then" is reserved.
- */
- protected abstract boolean isReserved(String name);
-
private Token lookAhead(int distance) {
// Read in as many as needed.
while (distance >= mRead.size()) {
@@ -200,7 +159,6 @@ private Token lookAhead(int distance) {
}
private final TokenReader mTokens;
-
private final List<Token> mRead;
private final List<Token> mConsumed;
}
View
4 src/com/stuffwithstuff/magpie/parser/ParserTable.java
@@ -41,10 +41,6 @@ public T get(Token token) {
return mTypeTable.get(token.getType());
}
- public boolean isReserved(String name) {
- return mNameTable.containsKey(name);
- }
-
private final Map<String, T> mNameTable = new HashMap<String, T>();
private final Map<TokenType, T> mTypeTable = new HashMap<TokenType, T>();
}
View
2 src/com/stuffwithstuff/magpie/parser/PatternParser.java
@@ -72,7 +72,7 @@ private static Pattern record(MagpieParser parser) {
}
private static Pattern primary(MagpieParser parser) {
- if (parser.match("is")) {
+ if (parser.match(TokenType.IS)) {
Expr type = parser.parsePrecedence(Precedence.COMPARISON);
return Pattern.type(type);
} else if (parser.match(TokenType.EQEQ)) {
View
5 src/com/stuffwithstuff/magpie/parser/Token.java
@@ -30,11 +30,6 @@ public Token(Position position, TokenType type, String text, Object value) {
public double getDouble() { return ((Double)mValue).doubleValue(); }
public String getString() { return (String)mValue; }
- public boolean isKeyword(String name) {
- if (mType != TokenType.NAME) return false;
- return name.equals(getString());
- }
-
public String toString() {
return mText;
}
View
25 src/com/stuffwithstuff/magpie/parser/TokenType.java
@@ -27,7 +27,32 @@
// keywords
AND,
+ AS,
+ BREAK,
+ CASE,
+ CATCH,
+ DEF,
+ DEFCLASS,
+ DO,
+ ELSE,
+ END,
+ EXPORT,
+ FN,
+ FOR,
+ IF,
+ IMPORT,
+ IN,
+ IS,
+ MATCH,
+ ONLY,
OR,
+ RETURN,
+ THEN,
+ THROW,
+ VAL,
+ VAR,
+ WHILE,
+ WITH,
// identifiers
NAME,

0 comments on commit 17aa423

Please sign in to comment.
Something went wrong with that request. Please try again.