Cleaned up a lot of code by using some options from JavaCC

javaparser-core/src/main/java/com/github/javaparser/TokenBase.java

@@ -0,0 +1,8 @@
+package com.github.javaparser;
+
+import static com.github.javaparser.GeneratedJavaParserConstants.GT;
+
+class TokenBase {
+    int realKind = GT;
+    JavaToken javaToken;
+}

javaparser-core/src/main/javacc-support/com/github/javaparser/GeneratedJavaParserBase.java

@@ -67,22 +67,6 @@ void addProblem(String message) {
         problems.add(new Problem(message, tokenRange(), null));
     }
 
-    /* Supports a case where >> should be two tokens instead of one,
-        and keeps track of the JavaParser specific token type for this token */
-    static final class CustomToken extends Token {
-        int realKind = GT;
-        JavaToken javaToken;
-
-        CustomToken(int kind, String image) {
-            this.kind = kind;
-            this.image = image;
-        }
-
-        public static Token newToken(int kind, String image) {
-            return new CustomToken(kind, image);
-        }
-    }
-
     /* Returns a tokenRange that spans the last matched token */
     TokenRange tokenRange() {
         if(storeTokens) {
@@ -182,9 +166,7 @@ void setStoreTokens(boolean storeTokens) {
     TokenRange recover(int recoveryTokenType, ParseException p) {
         JavaToken begin = null;
         if (p.currentToken != null) {
-            if (p.currentToken instanceof CustomToken) {
-                begin = token();
-            }
+            begin = token();
         }
         Token t;
         do {
@@ -390,11 +372,11 @@ private String makeMessageForParseException(ParseException exception) {
      * Create a TokenRange that spans exactly one token
      */
     private static TokenRange tokenRange(Token token) {
-        JavaToken javaToken = ((CustomToken) token).javaToken;
+        JavaToken javaToken = token.javaToken;
         return new TokenRange(javaToken, javaToken);
     }
 
-    static Comment createCommentFromToken(CustomToken token) {
+    static Comment createCommentFromToken(Token token) {
         String commentText = token.image;
         if (token.kind == JAVADOC_COMMENT) {
             return new JavadocComment(tokenRange(token), commentText.substring(3, commentText.length() - 2));

javaparser-core/src/main/javacc/java.jj

@@ -20,12 +20,11 @@
  */
 
 options {
-  LOOKAHEAD=1;
   STATIC=false;
   JAVA_UNICODE_ESCAPE=true;
   COMMON_TOKEN_ACTION=true;
   JDK_VERSION = "1.8";
-  TOKEN_FACTORY = "GeneratedJavaParser.CustomToken";
+  TOKEN_EXTENDS ="TokenBase";
   JAVA_TEMPLATE_TYPE = "modern";
 }
 
@@ -70,7 +69,7 @@ import static com.github.javaparser.ast.type.ArrayType.*;
 final class GeneratedJavaParser extends GeneratedJavaParserBase {
     /* Returns the JavaParser specific token type of the last matched token */
     JavaToken token() {
-        return ((CustomToken)token).javaToken;
+        return token.javaToken;
     }
 
     /* Changes the amount by which the horizontal position is increased when a tab character is encountered.
@@ -103,7 +102,7 @@ TOKEN_MGR_DECLS :
     private List<JavaToken> tokens = new ArrayList<JavaToken>();
     private CommentsCollection commentsCollection = new CommentsCollection();
     private JavaToken homeToken;
-    private Stack<CustomToken> tokenWorkStack = new Stack<CustomToken>();
+    private Stack<Token> tokenWorkStack = new Stack<Token>();
     private boolean storeTokens;
 
     void reset() {
@@ -133,12 +132,11 @@ TOKEN_MGR_DECLS :
         this.storeTokens = storeTokens;
     }
 
-    private void CommonTokenAction(Token rawToken) {
+    private void CommonTokenAction(Token token) {
         // Use an intermediary stack to avoid recursion, see issue 1003
-        CustomToken token = (CustomToken) rawToken;
         do {
             tokenWorkStack.push(token);
-            token = (CustomToken) token.specialToken;
+            token = token.specialToken;
         } while (token != null);
 
         // The stack is now filled with tokens in left-to-right order. Process them.
@@ -576,13 +574,13 @@ TOKEN :
   < RUNSIGNEDSHIFT: ">>>" >
   {
      matchedToken.kind = GT;
-     ((GeneratedJavaParser.CustomToken)matchedToken).realKind = RUNSIGNEDSHIFT;
+     matchedToken.realKind = RUNSIGNEDSHIFT;
      input_stream.backup(2);
   }
 | < RSIGNEDSHIFT: ">>" >
   {
      matchedToken.kind = GT;
-     ((GeneratedJavaParser.CustomToken)matchedToken).realKind = RSIGNEDSHIFT;
+     matchedToken.realKind = RSIGNEDSHIFT;
      input_stream.backup(1);
   }
 | < GT: ">" >
@@ -2491,7 +2489,7 @@ void RUNSIGNEDSHIFT():
 {}
 {
   ( LOOKAHEAD({ getToken(1).kind == GT &&
-                ((CustomToken)getToken(1)).realKind == RUNSIGNEDSHIFT} )
+                getToken(1).realKind == RUNSIGNEDSHIFT} )
    ">" ">" ">"
   )
 }
@@ -2500,7 +2498,7 @@ void RSIGNEDSHIFT():
 {}
 {
   ( LOOKAHEAD({ getToken(1).kind == GT &&
-                ((CustomToken)getToken(1)).realKind == RSIGNEDSHIFT} )
+                getToken(1).realKind == RSIGNEDSHIFT} )
   ">" ">"
   )
 }