Reorganize a little more

javaparser-core/src/main/javacc-support/com/github/javaparser/GeneratedJavaParserBase.java

@@ -5,7 +5,7 @@
 import com.github.javaparser.ast.Node;
 import com.github.javaparser.ast.NodeList;
 import com.github.javaparser.ast.body.Parameter;
-import com.github.javaparser.ast.comments.*;
+import com.github.javaparser.ast.comments.CommentsCollection;
 import com.github.javaparser.ast.expr.*;
 import com.github.javaparser.ast.stmt.Statement;
 import com.github.javaparser.ast.type.ArrayType;
@@ -15,8 +15,7 @@
 
 import java.util.*;
 
-import static com.github.javaparser.GeneratedJavaParserConstants.*;
-import static com.github.javaparser.Position.pos;
+import static com.github.javaparser.GeneratedJavaParserConstants.EOF;
 import static com.github.javaparser.ast.type.ArrayType.unwrapArrayTypes;
 import static com.github.javaparser.ast.type.ArrayType.wrapInArrayTypes;
 import static com.github.javaparser.utils.Utils.assertNotNull;
@@ -365,34 +364,4 @@ private String makeMessageForParseException(ParseException exception) {
         }
         return sb.toString();
     }
-
-    ///// These are for the token manager that can't get a superclass until javacc 7.
-
-    /**
-     * Create a TokenRange that spans exactly one token
-     */
-    private static TokenRange tokenRange(Token token) {
-        JavaToken javaToken = token.javaToken;
-        return new TokenRange(javaToken, javaToken);
-    }
-
-    static Comment createCommentFromToken(Token token) {
-        String commentText = token.image;
-        if (token.kind == JAVADOC_COMMENT) {
-            return new JavadocComment(tokenRange(token), commentText.substring(3, commentText.length() - 2));
-        } else if (token.kind == MULTI_LINE_COMMENT) {
-            return new BlockComment(tokenRange(token), commentText.substring(2, commentText.length() - 2));
-        } else if (token.kind == SINGLE_LINE_COMMENT) {
-            // line comments have their end of line character(s) included, and we don't want that.
-            Range range = new Range(pos(token.beginLine, token.beginColumn), pos(token.endLine, token.endColumn));
-            while (commentText.endsWith("\r") || commentText.endsWith("\n")) {
-                commentText = commentText.substring(0, commentText.length() - 1);
-            }
-            range = range.withEnd(pos(range.begin.line, range.begin.column + commentText.length()));
-            LineComment comment = new LineComment(tokenRange(token), commentText.substring(2));
-            comment.setRange(range);
-            return comment;
-        }
-        throw new AssertionError("Unexpectedly got passed a non-comment token.");
-    }
 }

javaparser-core/src/main/javacc-support/com/github/javaparser/GeneratedJavaParserTokenManagerBase.java

@@ -0,0 +1,46 @@
+package com.github.javaparser;
+
+import com.github.javaparser.ast.comments.BlockComment;
+import com.github.javaparser.ast.comments.Comment;
+import com.github.javaparser.ast.comments.JavadocComment;
+import com.github.javaparser.ast.comments.LineComment;
+
+import static com.github.javaparser.GeneratedJavaParserConstants.JAVADOC_COMMENT;
+import static com.github.javaparser.GeneratedJavaParserConstants.MULTI_LINE_COMMENT;
+import static com.github.javaparser.GeneratedJavaParserConstants.SINGLE_LINE_COMMENT;
+import static com.github.javaparser.Position.pos;
+
+/** Base class for {@link com.github.javaparser.GeneratedJavaParserTokenManager} */
+public class GeneratedJavaParserTokenManagerBase {
+    /**
+     * Create a TokenRange that spans exactly one token
+     */
+    private static TokenRange tokenRange(Token token) {
+        JavaToken javaToken = token.javaToken;
+        return new TokenRange(javaToken, javaToken);
+    }
+
+    /** 
+     * Since comments are completely captured in a single token, including their delimiters, 
+     * deconstruct them here so we can turn them into nodes later on.
+     */
+    static Comment createCommentFromToken(Token token) {
+        String commentText = token.image;
+        if (token.kind == JAVADOC_COMMENT) {
+            return new JavadocComment(tokenRange(token), commentText.substring(3, commentText.length() - 2));
+        } else if (token.kind == MULTI_LINE_COMMENT) {
+            return new BlockComment(tokenRange(token), commentText.substring(2, commentText.length() - 2));
+        } else if (token.kind == SINGLE_LINE_COMMENT) {
+            // line comments have their end of line character(s) included, and we don't want that.
+            Range range = new Range(pos(token.beginLine, token.beginColumn), pos(token.endLine, token.endColumn));
+            while (commentText.endsWith("\r") || commentText.endsWith("\n")) {
+                commentText = commentText.substring(0, commentText.length() - 1);
+            }
+            range = range.withEnd(pos(range.begin.line, range.begin.column + commentText.length()));
+            LineComment comment = new LineComment(tokenRange(token), commentText.substring(2));
+            comment.setRange(range);
+            return comment;
+        }
+        throw new AssertionError("Unexpectedly got passed a non-comment token.");
+    }
+}

javaparser-core/src/main/javacc-support/com/github/javaparser/TokenBase.java

@@ -0,0 +1,11 @@
+package com.github.javaparser;
+
+import static com.github.javaparser.GeneratedJavaParserConstants.GT;
+
+/** Base class for the generated {@link Token} */
+class TokenBase {
+    /** For tracking the >> >>> ambiguity. */
+    int realKind = GT;
+    /** This is the link to the token that JavaParser presents to the user */
+    JavaToken javaToken;
+}

javaparser-core/src/main/javacc/java.jj

@@ -65,6 +65,8 @@ import static com.github.javaparser.Range.*;
 import static com.github.javaparser.Position.*;
 import static com.github.javaparser.ast.type.ArrayType.*;
 
+import static com.github.javaparser.GeneratedJavaParserTokenManagerBase.*;
+
 @Generated("JavaCC")
 final class GeneratedJavaParser extends GeneratedJavaParserBase {
     /* Returns the JavaParser specific token type of the last matched token */