diff --git a/javaparser-core-generators/src/main/java/com/github/javaparser/generator/NodeGenerator.java b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/NodeGenerator.java index 673990df26..18ab60d999 100644 --- a/javaparser-core-generators/src/main/java/com/github/javaparser/generator/NodeGenerator.java +++ b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/NodeGenerator.java @@ -32,9 +32,9 @@ public final void generate() throws Exception { after(); } - protected Pair parseNode(BaseNodeMetaModel nodeMetaModel) throws IOException { + protected Pair parseNode(BaseNodeMetaModel nodeMetaModel) { CompilationUnit nodeCu = sourceRoot.parse(nodeMetaModel.getPackageName(), nodeMetaModel.getTypeName() + ".java"); - ClassOrInterfaceDeclaration nodeCoid = nodeCu.getClassByName(nodeMetaModel.getTypeName()).orElseThrow(() -> new IOException("Can't find class")); + ClassOrInterfaceDeclaration nodeCoid = nodeCu.getClassByName(nodeMetaModel.getTypeName()).orElseThrow(() -> new AssertionError("Can't find class")); return new Pair<>(nodeCu, nodeCoid); } diff --git a/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/CoreGenerator.java b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/CoreGenerator.java index 2618b0f17e..c1aa5f9fbf 100644 --- a/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/CoreGenerator.java +++ b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/CoreGenerator.java @@ -2,6 +2,7 @@ import com.github.javaparser.ParserConfiguration; import com.github.javaparser.generator.core.node.*; +import com.github.javaparser.generator.core.other.TokenKindGenerator; import com.github.javaparser.generator.core.visitor.*; import com.github.javaparser.utils.Log; import com.github.javaparser.utils.SourceRoot; @@ -11,28 +12,37 @@ /** * Generates all generated visitors in the javaparser-core module. + * Suggested usage is by running the run_core_generators.sh script. + * You may want to run_metamodel_generator.sh before that. */ public class CoreGenerator { + private static final ParserConfiguration parserConfiguration = new ParserConfiguration() +// .setStoreTokens(false) +// .setAttributeComments(false) +// .setLexicalPreservationEnabled(true) + ; + public static void main(String[] args) throws Exception { if (args.length != 1) { throw new RuntimeException("Need 1 parameter: the JavaParser source checkout root directory."); } Log.setAdapter(new Log.StandardOutStandardErrorAdapter()); final Path root = Paths.get(args[0], "..", "javaparser-core", "src", "main", "java"); - final SourceRoot sourceRoot = new SourceRoot(root) + final SourceRoot sourceRoot = new SourceRoot(root, parserConfiguration) // .setPrinter(LexicalPreservingPrinter::print) - .setParserConfiguration(new ParserConfiguration() -// .setStoreTokens(false) -// .setAttributeComments(false) -// .setLexicalPreservationEnabled(true) - ); + ; + + final Path generatedJavaCcRoot = Paths.get(args[0], "..", "javaparser-core", "target", "generated-sources", "javacc"); + final SourceRoot generatedJavaCcSourceRoot = new SourceRoot(generatedJavaCcRoot, parserConfiguration) +// .setPrinter(LexicalPreservingPrinter::print) + ; - new CoreGenerator().run(sourceRoot); + new CoreGenerator().run(sourceRoot, generatedJavaCcSourceRoot); sourceRoot.saveAll(); } - private void run(SourceRoot sourceRoot) throws Exception { + private void run(SourceRoot sourceRoot, SourceRoot generatedJavaCcSourceRoot) throws Exception { new TypeCastingGenerator(sourceRoot).generate(); new GenericListVisitorAdapterGenerator(sourceRoot).generate(); new GenericVisitorAdapterGenerator(sourceRoot).generate(); @@ -58,5 +68,6 @@ private void run(SourceRoot sourceRoot) throws Exception { new MainConstructorGenerator(sourceRoot).generate(); new FinalGenerator(sourceRoot).generate(); new AcceptGenerator(sourceRoot).generate(); + new TokenKindGenerator(sourceRoot, generatedJavaCcSourceRoot).generate(); } } diff --git a/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/other/TokenKindGenerator.java b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/other/TokenKindGenerator.java new file mode 100644 index 0000000000..5088bc3536 --- /dev/null +++ b/javaparser-core-generators/src/main/java/com/github/javaparser/generator/core/other/TokenKindGenerator.java @@ -0,0 +1,71 @@ +package com.github.javaparser.generator.core.other; + +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.NodeList; +import com.github.javaparser.ast.body.BodyDeclaration; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.EnumConstantDeclaration; +import com.github.javaparser.ast.body.EnumDeclaration; +import com.github.javaparser.ast.expr.IntegerLiteralExpr; +import com.github.javaparser.ast.stmt.ReturnStmt; +import com.github.javaparser.ast.stmt.SwitchEntryStmt; +import com.github.javaparser.ast.stmt.SwitchStmt; +import com.github.javaparser.generator.Generator; +import com.github.javaparser.utils.Log; +import com.github.javaparser.utils.SourceRoot; + +/** + * Generates the TokenKind enum from {@link com.github.javaparser.GeneratedJavaParserConstants} + */ +public class TokenKindGenerator extends Generator { + private final SourceRoot generatedJavaCcSourceRoot; + + public TokenKindGenerator(SourceRoot sourceRoot, SourceRoot generatedJavaCcSourceRoot) { + super(sourceRoot); + this.generatedJavaCcSourceRoot = generatedJavaCcSourceRoot; + } + + @Override + public void generate() { + Log.info("Running %s", getClass().getSimpleName()); + + final CompilationUnit javaTokenCu = sourceRoot.parse("com.github.javaparser", "JavaToken.java"); + final ClassOrInterfaceDeclaration javaToken = javaTokenCu.getClassByName("JavaToken").orElseThrow(() -> new AssertionError("Can't find class in java file.")); + final EnumDeclaration kindEnum = javaToken.findFirst(EnumDeclaration.class, e -> e.getNameAsString().equals("Kind")).orElseThrow(() -> new AssertionError("Can't find class in java file.")); + + kindEnum.getEntries().clear(); + annotateGenerated(kindEnum); + + final SwitchStmt valueOfSwitch = kindEnum.findFirst(SwitchStmt.class).orElseThrow(() -> new AssertionError("Can't find valueOf switch.")); + valueOfSwitch.findAll(SwitchEntryStmt.class).stream().filter(e -> e.getLabel().isPresent()).forEach(Node::remove); + + final CompilationUnit constantsCu = generatedJavaCcSourceRoot.parse("com.github.javaparser", "GeneratedJavaParserConstants.java"); + final ClassOrInterfaceDeclaration constants = constantsCu.getInterfaceByName("GeneratedJavaParserConstants").orElseThrow(() -> new AssertionError("Can't find class in java file.")); + for (BodyDeclaration member : constants.getMembers()) { + member.toFieldDeclaration() + .filter(field -> { + String javadoc = field.getJavadocComment().get().getContent(); + return javadoc.contains("RegularExpression Id") || javadoc.contains("End of File"); + }) + .map(field -> field.getVariable(0)) + .ifPresent(var -> { + final String name = var.getNameAsString(); + final IntegerLiteralExpr kind = var.getInitializer().get().asIntegerLiteralExpr(); + generateEnumEntry(kindEnum, name, kind); + generateValueOfEntry(valueOfSwitch, name, kind); + }); + } + } + + private void generateValueOfEntry(SwitchStmt valueOfSwitch, String name, IntegerLiteralExpr kind) { + final SwitchEntryStmt entry = new SwitchEntryStmt(kind, new NodeList<>(new ReturnStmt(name))); + valueOfSwitch.getEntries().addFirst(entry); + } + + private void generateEnumEntry(EnumDeclaration kindEnum, String name, IntegerLiteralExpr kind) { + final EnumConstantDeclaration enumEntry = new EnumConstantDeclaration(name); + enumEntry.getArguments().add(kind); + kindEnum.addEntry(enumEntry); + } +} diff --git a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java index 2e103cfbbc..94af706d9d 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java +++ b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java @@ -18,15 +18,14 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. */ - package com.github.javaparser; import java.util.List; import java.util.Optional; - import static com.github.javaparser.utils.CodeGenerationUtils.f; import static com.github.javaparser.utils.Utils.EOL; import static com.github.javaparser.utils.Utils.assertNotNull; +import javax.annotation.Generated; /** * A token from a parsed source file. @@ -34,12 +33,17 @@ * It is a node in a double linked list called token list. */ public class JavaToken { + public static final JavaToken INVALID = new JavaToken(); private Range range; + private int kind; + private String text; + private JavaToken previousToken = null; + private JavaToken nextToken = null; private JavaToken() { @@ -53,39 +57,6 @@ public JavaToken(int kind, String text) { JavaToken(Token token, List tokens) { Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn); String text = token.image; - - // You could be puzzled by the following lines - // - // The reason why these lines are necessary is the fact that Java is ambiguous. There are cases where the - // sequence of characters ">>>" and ">>" should be recognized as the single tokens ">>>" and ">>". In other - // cases however we want to split those characters in single GT tokens (">"). - // - // For example, in expressions ">>" and ">>>" are valid, while when defining types we could have this: - // - // List>>> - // - // You can see that the sequence ">>>>" should be interpreted as four consecutive ">" tokens closing a type - // parameter list. - // - // The JavaCC handle this case by first recognizing always the longest token, and then depending on the context - // putting back the unused chars in the stream. However in those cases the token provided is invalid: it has an - // image corresponding to the text originally recognized, without considering that after some characters could - // have been put back into the stream. - // - // So in the case of: - // - // List>>> - // ___ -> recognized as ">>>", then ">>" put back in the stream but - // Token(type=GT, image=">>>") passed to this class - // ___ -> recognized as ">>>", then ">>" put back in the stream but - // Token(type=GT, image=">>>") passed to this class - // __ -> recognized as ">>", then ">" put back in the stream but - // Token(type=GT, image=">>") passed to this class - // _ -> Token(type=GT, image=">") good! - // - // So given the image could be wrong but the type is correct, we look at the type of the token and we fix - // the image. Everybody is happy and we can keep this horrible thing as our little secret. - if (token.kind == GeneratedJavaParserConstants.GT) { range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn); text = ">"; @@ -122,10 +93,8 @@ public JavaToken(int kind) { this.text = content; } - public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) { assertNotNull(text); - this.range = range; this.kind = kind; this.text = text; @@ -178,16 +147,8 @@ public TokenRange toTokenRange() { @Override public String toString() { - String text = getText() - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\r\n", "\\r\\n") - .replace("\t", "\\t") - ; - return f("\"%s\" <%s> %s", - text, - getKind(), - getRange().map(Range::toString).orElse("(?)-(?)")); + String text = getText().replace("\n", "\\n").replace("\r", "\\r").replace("\r\n", "\\r\\n").replace("\t", "\\t"); + return f("\"%s\" <%s> %s", text, getKind(), getRange().map(Range::toString).orElse("(?)-(?)")); } /** @@ -205,6 +166,7 @@ public boolean invalid() { } public enum Category { + WHITESPACE_NO_EOL, EOL, COMMENT, IDENTIFIER, KEYWORD, LITERAL, SEPARATOR, OPERATOR; public boolean isWhitespaceOrComment() { @@ -248,6 +210,315 @@ public boolean isOperator() { } } + @Generated("com.github.javaparser.generator.core.other.TokenKindGenerator") + public enum Kind { + + EOF(0), SPACE(1), WINDOWS_EOL(2), UNIX_EOL(3), OLD_MAC_EOL(4), SINGLE_LINE_COMMENT(5), ENTER_JAVADOC_COMMENT(6), ENTER_MULTILINE_COMMENT(7), JAVADOC_COMMENT(8), MULTI_LINE_COMMENT(9), COMMENT_CONTENT(10), ABSTRACT(11), ASSERT(12), BOOLEAN(13), BREAK(14), BYTE(15), CASE(16), CATCH(17), CHAR(18), CLASS(19), CONST(20), CONTINUE(21), _DEFAULT(22), DO(23), DOUBLE(24), ELSE(25), ENUM(26), EXTENDS(27), FALSE(28), FINAL(29), FINALLY(30), FLOAT(31), FOR(32), GOTO(33), IF(34), IMPLEMENTS(35), IMPORT(36), INSTANCEOF(37), INT(38), INTERFACE(39), LONG(40), NATIVE(41), NEW(42), NULL(43), PACKAGE(44), PRIVATE(45), PROTECTED(46), PUBLIC(47), RETURN(48), SHORT(49), STATIC(50), STRICTFP(51), SUPER(52), SWITCH(53), SYNCHRONIZED(54), THIS(55), THROW(56), THROWS(57), TRANSIENT(58), TRUE(59), TRY(60), VOID(61), VOLATILE(62), WHILE(63), REQUIRES(64), TO(65), WITH(66), OPEN(67), OPENS(68), USES(69), MODULE(70), EXPORTS(71), PROVIDES(72), TRANSITIVE(73), LONG_LITERAL(74), INTEGER_LITERAL(75), DECIMAL_LITERAL(76), HEX_LITERAL(77), OCTAL_LITERAL(78), BINARY_LITERAL(79), FLOATING_POINT_LITERAL(80), DECIMAL_FLOATING_POINT_LITERAL(81), DECIMAL_EXPONENT(82), HEXADECIMAL_FLOATING_POINT_LITERAL(83), HEXADECIMAL_EXPONENT(84), HEX_DIGITS(85), UNICODE_ESCAPE(86), CHARACTER_LITERAL(87), STRING_LITERAL(88), IDENTIFIER(89), LETTER(90), PART_LETTER(91), LPAREN(92), RPAREN(93), LBRACE(94), RBRACE(95), LBRACKET(96), RBRACKET(97), SEMICOLON(98), COMMA(99), DOT(100), AT(101), ASSIGN(102), LT(103), BANG(104), TILDE(105), HOOK(106), COLON(107), EQ(108), LE(109), GE(110), NE(111), SC_OR(112), SC_AND(113), INCR(114), DECR(115), PLUS(116), MINUS(117), STAR(118), SLASH(119), BIT_AND(120), BIT_OR(121), XOR(122), REM(123), LSHIFT(124), PLUSASSIGN(125), MINUSASSIGN(126), STARASSIGN(127), SLASHASSIGN(128), ANDASSIGN(129), ORASSIGN(130), XORASSIGN(131), REMASSIGN(132), LSHIFTASSIGN(133), RSIGNEDSHIFTASSIGN(134), RUNSIGNEDSHIFTASSIGN(135), ELLIPSIS(136), ARROW(137), DOUBLECOLON(138), RUNSIGNEDSHIFT(139), RSIGNEDSHIFT(140), GT(141), CTRL_Z(142); + + private final int kind; + + Kind(int kind) { + this.kind = kind; + } + + public Kind valueOf(int kind) { + switch(kind) { + case 142: + return CTRL_Z; + case 141: + return GT; + case 140: + return RSIGNEDSHIFT; + case 139: + return RUNSIGNEDSHIFT; + case 138: + return DOUBLECOLON; + case 137: + return ARROW; + case 136: + return ELLIPSIS; + case 135: + return RUNSIGNEDSHIFTASSIGN; + case 134: + return RSIGNEDSHIFTASSIGN; + case 133: + return LSHIFTASSIGN; + case 132: + return REMASSIGN; + case 131: + return XORASSIGN; + case 130: + return ORASSIGN; + case 129: + return ANDASSIGN; + case 128: + return SLASHASSIGN; + case 127: + return STARASSIGN; + case 126: + return MINUSASSIGN; + case 125: + return PLUSASSIGN; + case 124: + return LSHIFT; + case 123: + return REM; + case 122: + return XOR; + case 121: + return BIT_OR; + case 120: + return BIT_AND; + case 119: + return SLASH; + case 118: + return STAR; + case 117: + return MINUS; + case 116: + return PLUS; + case 115: + return DECR; + case 114: + return INCR; + case 113: + return SC_AND; + case 112: + return SC_OR; + case 111: + return NE; + case 110: + return GE; + case 109: + return LE; + case 108: + return EQ; + case 107: + return COLON; + case 106: + return HOOK; + case 105: + return TILDE; + case 104: + return BANG; + case 103: + return LT; + case 102: + return ASSIGN; + case 101: + return AT; + case 100: + return DOT; + case 99: + return COMMA; + case 98: + return SEMICOLON; + case 97: + return RBRACKET; + case 96: + return LBRACKET; + case 95: + return RBRACE; + case 94: + return LBRACE; + case 93: + return RPAREN; + case 92: + return LPAREN; + case 91: + return PART_LETTER; + case 90: + return LETTER; + case 89: + return IDENTIFIER; + case 88: + return STRING_LITERAL; + case 87: + return CHARACTER_LITERAL; + case 86: + return UNICODE_ESCAPE; + case 85: + return HEX_DIGITS; + case 84: + return HEXADECIMAL_EXPONENT; + case 83: + return HEXADECIMAL_FLOATING_POINT_LITERAL; + case 82: + return DECIMAL_EXPONENT; + case 81: + return DECIMAL_FLOATING_POINT_LITERAL; + case 80: + return FLOATING_POINT_LITERAL; + case 79: + return BINARY_LITERAL; + case 78: + return OCTAL_LITERAL; + case 77: + return HEX_LITERAL; + case 76: + return DECIMAL_LITERAL; + case 75: + return INTEGER_LITERAL; + case 74: + return LONG_LITERAL; + case 73: + return TRANSITIVE; + case 72: + return PROVIDES; + case 71: + return EXPORTS; + case 70: + return MODULE; + case 69: + return USES; + case 68: + return OPENS; + case 67: + return OPEN; + case 66: + return WITH; + case 65: + return TO; + case 64: + return REQUIRES; + case 63: + return WHILE; + case 62: + return VOLATILE; + case 61: + return VOID; + case 60: + return TRY; + case 59: + return TRUE; + case 58: + return TRANSIENT; + case 57: + return THROWS; + case 56: + return THROW; + case 55: + return THIS; + case 54: + return SYNCHRONIZED; + case 53: + return SWITCH; + case 52: + return SUPER; + case 51: + return STRICTFP; + case 50: + return STATIC; + case 49: + return SHORT; + case 48: + return RETURN; + case 47: + return PUBLIC; + case 46: + return PROTECTED; + case 45: + return PRIVATE; + case 44: + return PACKAGE; + case 43: + return NULL; + case 42: + return NEW; + case 41: + return NATIVE; + case 40: + return LONG; + case 39: + return INTERFACE; + case 38: + return INT; + case 37: + return INSTANCEOF; + case 36: + return IMPORT; + case 35: + return IMPLEMENTS; + case 34: + return IF; + case 33: + return GOTO; + case 32: + return FOR; + case 31: + return FLOAT; + case 30: + return FINALLY; + case 29: + return FINAL; + case 28: + return FALSE; + case 27: + return EXTENDS; + case 26: + return ENUM; + case 25: + return ELSE; + case 24: + return DOUBLE; + case 23: + return DO; + case 22: + return _DEFAULT; + case 21: + return CONTINUE; + case 20: + return CONST; + case 19: + return CLASS; + case 18: + return CHAR; + case 17: + return CATCH; + case 16: + return CASE; + case 15: + return BYTE; + case 14: + return BREAK; + case 13: + return BOOLEAN; + case 12: + return ASSERT; + case 11: + return ABSTRACT; + case 10: + return COMMENT_CONTENT; + case 9: + return MULTI_LINE_COMMENT; + case 8: + return JAVADOC_COMMENT; + case 7: + return ENTER_MULTILINE_COMMENT; + case 6: + return ENTER_JAVADOC_COMMENT; + case 5: + return SINGLE_LINE_COMMENT; + case 4: + return OLD_MAC_EOL; + case 3: + return UNIX_EOL; + case 2: + return WINDOWS_EOL; + case 1: + return SPACE; + case 0: + return EOF; + default: + throw new IllegalArgumentException(f("Token kind %i is unknown.", kind)); + } + } + + public int getKind() { + return kind; + } + } + public JavaToken.Category getCategory() { return TokenTypes.getCategory(kind); } @@ -284,7 +555,6 @@ public void insertAfter(JavaToken newToken) { public void deleteToken() { final Optional nextToken = getNextToken(); final Optional previousToken = getPreviousToken(); - previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null)); nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null)); } @@ -335,14 +605,15 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; JavaToken javaToken = (JavaToken) o; - - if (kind != javaToken.kind) return false; - if (!text.equals(javaToken.text)) return false; - + if (kind != javaToken.kind) + return false; + if (!text.equals(javaToken.text)) + return false; return true; } } diff --git a/javaparser-core/src/main/java/com/github/javaparser/ast/NodeList.java b/javaparser-core/src/main/java/com/github/javaparser/ast/NodeList.java index 1ceb1b700e..3fd0a44195 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/ast/NodeList.java +++ b/javaparser-core/src/main/java/com/github/javaparser/ast/NodeList.java @@ -53,11 +53,7 @@ public class NodeList implements List, Iterable, HasParent private List observers = new ArrayList<>(); public NodeList() { - this((Node) null); - } - - public NodeList(Node parent) { - setParentNode(parent); + parentNode = null; } public NodeList(Collection n) {