From d1d5de5e5c259c8429aae19b2e54a0d1956cc68e Mon Sep 17 00:00:00 2001 From: Danny van Bruggen Date: Sat, 12 Aug 2017 22:15:44 +0200 Subject: [PATCH 1/2] Name two unnamed tokens --- javaparser-core/src/main/javacc/java.jj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javaparser-core/src/main/javacc/java.jj b/javaparser-core/src/main/javacc/java.jj index 71ebd97b96..d9d807fbfc 100644 --- a/javaparser-core/src/main/javacc/java.jj +++ b/javaparser-core/src/main/javacc/java.jj @@ -266,7 +266,7 @@ SPECIAL_TOKEN : MORE : { - <"/**" ~["/"]> { input_stream.backup(1); } : IN_JAVA_DOC_COMMENT + { input_stream.backup(1); } : IN_JAVA_DOC_COMMENT | <"/*"> : IN_MULTI_LINE_COMMENT } @@ -286,7 +286,7 @@ SPECIAL_TOKEN : MORE : { - < ~[] > + } /* RESERVED WORDS AND LITERALS */ From 3608acd141a4d3c2105672b4b19b26e4d2badb5c Mon Sep 17 00:00:00 2001 From: Danny van Bruggen Date: Sat, 12 Aug 2017 22:53:47 +0200 Subject: [PATCH 2/2] Make token list manipulatable --- .../java/com/github/javaparser/JavaToken.java | 150 +++++++++++++++--- .../java/com/github/javaparser/Problem.java | 9 +- .../com/github/javaparser/TokenRange.java | 7 +- .../java/com/github/javaparser/ast/Node.java | 5 +- .../github/javaparser/ast/type/ArrayType.java | 2 +- .../LexicalPreservingPrinter.java | 12 +- .../com/github/javaparser/JavaParserTest.java | 2 +- .../com/github/javaparser/JavaTokenTest.java | 10 +- .../com/github/javaparser/ProblemTest.java | 2 +- .../javaparser/ast/expr/LambdaExprTest.java | 4 +- 10 files changed, 165 insertions(+), 38 deletions(-) diff --git a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java index eb0b3058d0..ab774b1afd 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java +++ b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java @@ -24,30 +24,33 @@ import java.util.List; import java.util.Optional; -import static com.github.javaparser.Position.pos; +import static com.github.javaparser.utils.CodeGenerationUtils.f; +import static com.github.javaparser.utils.Utils.EOL; import static com.github.javaparser.utils.Utils.assertNotNull; /** * A token from a parsed source file. * (Awkwardly named "Java"Token since JavaCC already generates an internal class Token.) + * It is a node in a double linked list called token list. */ public class JavaToken { public static final JavaToken INVALID = new JavaToken(); - private final Range range; + private Range range; private int kind; - private final String text; - private final Optional previousToken; - private Optional nextToken = Optional.empty(); + private String text; + private JavaToken previousToken = null; + private JavaToken nextToken = null; private JavaToken() { - range = new Range(pos(-1, -1), pos(-1, -1)); - kind = 0; - text = "INVALID"; - previousToken = Optional.empty(); + this(null, 0, "INVALID", null, null); } - public JavaToken(Token token, List tokens) { + public JavaToken(int kind, String text) { + this(null, kind, text, null, null); + } + + JavaToken(Token token, List tokens) { Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn); String text = token.image; @@ -95,14 +98,34 @@ public JavaToken(Token token, List tokens) { this.text = text; if (!tokens.isEmpty()) { final JavaToken previousToken = tokens.get(tokens.size() - 1); - this.previousToken = Optional.of(previousToken); - previousToken.nextToken = Optional.of(this); + this.previousToken = previousToken; + previousToken.nextToken = this; } else { - previousToken = Optional.empty(); + previousToken = null; + } + } + + /** + * Create a token of a certain kind. + */ + public JavaToken(int kind) { + String content = GeneratedJavaParserConstants.tokenImage[kind]; + if (content.startsWith("\"")) { + content = content.substring(1, content.length() - 1); + } + if (TokenTypes.isEndOfLineToken(kind)) { + content = EOL; + } else if (TokenTypes.isWhitespace(kind)) { + content = " "; } + this.kind = kind; + this.text = content; } - public JavaToken(Range range, int kind, String text, Optional previousToken, Optional nextToken) { + + public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) { + assertNotNull(text); + this.range = range; this.kind = kind; this.text = text; @@ -110,8 +133,8 @@ public JavaToken(Range range, int kind, String text, Optional previou this.nextToken = nextToken; } - public Range getRange() { - return range; + public Optional getRange() { + return Optional.ofNullable(range); } public int getKind() { @@ -127,16 +150,31 @@ public String getText() { } public Optional getNextToken() { - return nextToken; + return Optional.ofNullable(nextToken); } public Optional getPreviousToken() { - return previousToken; + return Optional.ofNullable(previousToken); + } + + public void setRange(Range range) { + this.range = range; + } + + public void setText(String text) { + this.text = text; + } + + public String asString() { + return text; } @Override public String toString() { - return text; + return f("\"%s\" <%s> %s", + getText(), + getKind(), + getRange().map(Range::toString).orElse("(?)-(?)")); } /** @@ -211,4 +249,78 @@ public boolean isOperator() { public JavaToken.Category getCategory() { return TokenTypes.getCategory(kind); } + + /** + * Inserts newToken into the token list just before this token. + */ + public void insert(JavaToken newToken) { + assertNotNull(newToken); + getPreviousToken().ifPresent(p -> { + p.nextToken = newToken; + newToken.previousToken = p; + }); + previousToken = newToken; + newToken.nextToken = this; + } + + /** + * Inserts newToken into the token list just after this token. + */ + public void insertAfter(JavaToken newToken) { + assertNotNull(newToken); + getNextToken().ifPresent(n -> { + n.previousToken = newToken; + newToken.nextToken = n; + }); + nextToken = newToken; + newToken.previousToken = this; + } + + /** + * Links the tokens around the current token together, making the current token disappear from the list. + */ + public void deleteToken() { + final Optional nextToken = getNextToken(); + final Optional previousToken = getPreviousToken(); + + previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null)); + nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null)); + } + + /** + * Replaces the current token with newToken. + */ + public void replaceToken(JavaToken newToken) { + assertNotNull(newToken); + getPreviousToken().ifPresent(p -> { + p.nextToken = newToken; + newToken.previousToken = p; + }); + getNextToken().ifPresent(n -> { + n.previousToken = newToken; + newToken.nextToken = n; + }); + } + + /** + * @return the last token in the token list. + */ + public JavaToken findLastToken() { + JavaToken current = this; + while (current.getNextToken().isPresent()) { + current = current.getNextToken().get(); + } + return current; + } + + /** + * @return the first token in the token list. + */ + public JavaToken findFirstToken() { + JavaToken current = this; + while (current.getPreviousToken().isPresent()) { + current = current.getPreviousToken().get(); + } + return current; + } } diff --git a/javaparser-core/src/main/java/com/github/javaparser/Problem.java b/javaparser-core/src/main/java/com/github/javaparser/Problem.java index 156bb2f259..2fc0532267 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/Problem.java +++ b/javaparser-core/src/main/java/com/github/javaparser/Problem.java @@ -69,7 +69,7 @@ public String getMessage() { * @return the message plus location information. */ public String getVerboseMessage() { - return getLocation().map(l -> l.getBegin().getRange().begin + " " + message).orElse(message); + return getLocation().map(l -> l.getBegin().getRange().map(r -> r.begin.toString()).orElse("(line ?,col ?)") + " " + message).orElse(message); } /** @@ -98,8 +98,11 @@ public Optional getCause() { * Sorts problems on position. */ public static Comparator PROBLEM_BY_BEGIN_POSITION = (a, b) -> { - if (a.getLocation().isPresent() && b.getLocation().isPresent()) { - return a.getLocation().get().getBegin().getRange().begin.compareTo(b.getLocation().get().getBegin().getRange().begin); + final Optional aBegin= a.getLocation().flatMap(l -> l.getBegin().getRange().map(r -> r.begin)); + final Optional bBegin = b.getLocation().flatMap(l -> l.getBegin().getRange().map(r -> r.begin)); + + if (aBegin.isPresent() && bBegin.isPresent()) { + return aBegin.get().compareTo(bBegin.get()); } if (a.getLocation().isPresent() || b.getLocation().isPresent()) { if (a.getLocation().isPresent()) { diff --git a/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java b/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java index 7746e856d5..b6958eb0d2 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java +++ b/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java @@ -26,8 +26,11 @@ public JavaToken getEnd() { return end; } - public Range toRange() { - return new Range(begin.getRange().begin, end.getRange().end); + public Optional toRange() { + if (begin.getRange().isPresent() && end.getRange().isPresent()) { + return Optional.of(new Range(begin.getRange().get().begin, end.getRange().get().end)); + } + return Optional.empty(); } public TokenRange withBegin(JavaToken begin) { diff --git a/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java b/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java index d8630d9798..43263b16ed 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java +++ b/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java @@ -200,10 +200,11 @@ public Optional getTokenRange() { public Node setTokenRange(TokenRange tokenRange) { this.tokenRange = tokenRange; - if (tokenRange == null) { + if (tokenRange == null || + !(tokenRange.getBegin().getRange().isPresent() && tokenRange.getBegin().getRange().isPresent())) { range = null; } else { - range = new Range(tokenRange.getBegin().getRange().begin, tokenRange.getEnd().getRange().end); + range = new Range(tokenRange.getBegin().getRange().get().begin, tokenRange.getEnd().getRange().get().end); } return this; } diff --git a/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java b/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java index 37133ceccf..12e4ab46c1 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java +++ b/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java @@ -113,7 +113,7 @@ public static Type wrapInArrayTypes(Type type, List... arrayBr } type = new ArrayType(tokenRange, type, pair.getAnnotations()); if (tokenRange != null) { - type.setRange(tokenRange.toRange()); + type.setRange(tokenRange.toRange().get()); } } } diff --git a/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java b/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java index cb150bca71..8f719fdabb 100644 --- a/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java +++ b/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java @@ -202,11 +202,11 @@ public void process(Node node) { // We go over tokens and find to which nodes belong. Note that we start from the most specific nodes // and we move up to more general nodes for (JavaToken token : documentTokens) { - Optional maybeOwner = nodesDepthFirst.stream().filter(n -> n.getRange().get().contains(token.getRange())).findFirst(); - if (!maybeOwner.isPresent()) { - throw new RuntimeException("Token without node owning it: " + token); - } - Node owner = maybeOwner.get(); + Range tokenRange = token.getRange().orElseThrow(() -> new RuntimeException("Token without range: " + token)); + Node owner = nodesDepthFirst.stream() + .filter(n -> n.getRange().get().contains(tokenRange)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Token without node owning it: " + token)); if (!tokensByNode.containsKey(owner)) { tokensByNode.put(owner, new LinkedList<>()); } @@ -238,7 +238,7 @@ private void storeInitialTextForOneNode(Node node, List nodeTokens) { } } for (JavaToken token : nodeTokens) { - elements.add(new Pair<>(token.getRange(), new TokenTextElement(token))); + elements.add(new Pair<>(token.getRange().get(), new TokenTextElement(token))); } elements.sort(Comparator.comparing(e -> e.a.begin)); textForNodes.put(node, new NodeText(this, elements.stream().map(p -> p.b).collect(Collectors.toList()))); diff --git a/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java b/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java index 828e54ddb6..1082cd61f9 100644 --- a/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java +++ b/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java @@ -102,7 +102,7 @@ public void parseErrorContainsLocation() { ParseResult result = new JavaParser().parse(COMPILATION_UNIT, Providers.provider("class X { // blah")); Problem problem = result.getProblem(0); - assertEquals(range(1, 9, 1, 17), problem.getLocation().get().toRange()); + assertEquals(range(1, 9, 1, 17), problem.getLocation().get().toRange().get()); assertEquals("Parse error. Found , expected one of \";\" \"<\" \"@\" \"abstract\" \"boolean\" \"byte\" \"char\" \"class\" \"default\" \"double\" \"enum\" \"exports\" \"final\" \"float\" \"int\" \"interface\" \"long\" \"module\" \"native\" \"open\" \"opens\" \"private\" \"protected\" \"provides\" \"public\" \"requires\" \"short\" \"static\" \"strictfp\" \"synchronized\" \"to\" \"transient\" \"transitive\" \"uses\" \"void\" \"volatile\" \"with\" \"{\" \"}\" ", problem.getMessage()); assertInstanceOf(ParseException.class, problem.getCause().get()); } diff --git a/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java b/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java index c197e91a2c..0e38f6d3b0 100644 --- a/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java +++ b/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java @@ -53,11 +53,19 @@ public void testAFewTokens() { private void assertToken(String image, Range range, int kind, JavaToken.Category category, JavaToken token) { assertEquals(image, token.getText()); - assertEquals(range, token.getRange()); + assertEquals(range, token.getRange().get()); assertEquals(kind, token.getKind()); assertEquals(category, token.getCategory()); token.getNextToken().ifPresent(nt -> assertEquals(token, nt.getPreviousToken().get())); token.getPreviousToken().ifPresent(pt -> assertEquals(token, pt.getNextToken().get())); assertTrue(token.getNextToken().isPresent() || token.getPreviousToken().isPresent()); } + + @Test + public void testAFewImagesForTokenKinds() { + assertEquals("=", new JavaToken(ASSIGN).getText()); + // TODO this shouldn't be a space. + assertEquals(" ", new JavaToken(EOF).getText()); + assertEquals("*/", new JavaToken(JAVA_DOC_COMMENT).getText()); + } } diff --git a/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java b/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java index a26be5d5ee..0654344bac 100644 --- a/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java +++ b/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java @@ -20,7 +20,7 @@ public void testSimpleGetters() { public void testVerboseMessage() { Problem problem = new Problem("Parse error", TokenRange.INVALID, null); - assertEquals("(line -1,col -1) Parse error", problem.getVerboseMessage()); + assertEquals("(line ?,col ?) Parse error", problem.getVerboseMessage()); } @Test diff --git a/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java b/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java index 11b6dffe00..bfa007867f 100644 --- a/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java +++ b/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java @@ -22,7 +22,7 @@ public void lambdaRange2(){ private void assertRange(String startToken, String endToken, Node node) { TokenRange tokenRange = node.getTokenRange().get(); - assertEquals(startToken, tokenRange.getBegin().toString()); - assertEquals(endToken, tokenRange.getEnd().toString()); + assertEquals(startToken, tokenRange.getBegin().asString()); + assertEquals(endToken, tokenRange.getEnd().asString()); } } \ No newline at end of file