Skip to content

Commit

Permalink
Migrate lexing to javac
Browse files Browse the repository at this point in the history 
and complete the migration off ecj.

MOE_MIGRATED_REVID=139931395
  • Loading branch information
@cushon
cushon committed Nov 23, 2016
1 parent ce7852b commit 946e01c
Show file tree
Hide file tree
Showing 7 changed files with 292 additions and 106 deletions.
4 changes: 0 additions & 4 deletions core/pom.xml
View file
Expand Up @@ -39,10 +39,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>org.eclipse.jdt.core</artifactId>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>javac</artifactId>
Expand Down
21 changes: 6 additions & 15 deletions core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java
View file
Expand Up @@ -14,43 +14,34 @@
package com.google.googlejavaformat.java;

import static com.google.common.collect.Iterables.getLast;
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameclass;
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameenum;
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameinterface;

import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.googlejavaformat.Newlines;
import com.google.googlejavaformat.java.JavaInput.Tok;
import org.eclipse.jdt.core.compiler.InvalidInputException;
import com.sun.tools.javac.parser.Tokens.TokenKind;

/** Orders imports in Java source code. */
public class ImportOrderer {
/**
* Reorder the inputs in {@code input}, a complete Java program. On success, another complete Java
* Reorder the inputs in {@code text}, a complete Java program. On success, another complete Java
* program is returned, which is the same as the original except the imports are in order.
*
* @throws FormatterException if the input could not be parsed.
*/
public static String reorderImports(String text) throws FormatterException {
ImmutableList<Tok> toks;
try {
toks = JavaInput.buildToks(text, CLASS_START);
} catch (InvalidInputException e) {
// error handling is done during formatting
return text;
}
ImmutableList<Tok> toks = JavaInput.buildToks(text, CLASS_START);
return new ImportOrderer(text, toks).reorderImports();
}

/**
* Eclipse token ids that indicate the start of a type definition. We use this to avoid scanning
* {@link TokenKind}s that indicate the start of a type definition. We use this to avoid scanning
* the whole file, since we know that imports must precede any type definition.
*/
private static final ImmutableSet<Integer> CLASS_START =
ImmutableSet.of(TokenNameclass, TokenNameinterface, TokenNameenum);
private static final ImmutableSet<TokenKind> CLASS_START =
ImmutableSet.of(TokenKind.CLASS, TokenKind.INTERFACE, TokenKind.ENUM);

/**
* We use this set to find the first import, and again to check that there are no imports after
Expand Down
104 changes: 59 additions & 45 deletions core/src/main/java/com/google/googlejavaformat/java/JavaInput.java
View file
Expand Up @@ -16,6 +16,7 @@

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.getLast;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.MoreObjects;
import com.google.common.base.Verify;
Expand All @@ -31,15 +32,25 @@
import com.google.common.collect.TreeRangeSet;
import com.google.googlejavaformat.Input;
import com.google.googlejavaformat.Newlines;
import com.google.googlejavaformat.java.JavacTokens.RawTok;
import com.sun.tools.javac.file.JavacFileManager;
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
import com.sun.tools.javac.util.Context;
import com.sun.tools.javac.util.Log;
import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import org.eclipse.jdt.core.ToolFactory;
import org.eclipse.jdt.core.compiler.IScanner;
import org.eclipse.jdt.core.compiler.ITerminalSymbols;
import org.eclipse.jdt.core.compiler.InvalidInputException;
import javax.tools.Diagnostic;
import javax.tools.DiagnosticCollector;
import javax.tools.DiagnosticListener;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;

/** {@code JavaInput} extends {@link Input} to represent a Java input document. */
public final class JavaInput extends Input {
Expand All @@ -63,7 +74,7 @@ static final class Tok implements Input.Tok {
private final int position;
private final int columnI;
private final boolean isToken;
private final int id;
private final TokenKind kind;

/**
* The {@code Tok} constructor.
Expand All @@ -74,7 +85,7 @@ static final class Tok implements Input.Tok {
* @param position its {@code 0}-origin position in the input
* @param columnI its {@code 0}-origin column number in the input
* @param isToken whether the {@code Tok} is a token
* @param id the token id as defined by {@link org.eclipse.jdt.core.compiler.ITerminalSymbols}
* @param kind the token kind
*/
Tok(
int index,
Expand All @@ -83,14 +94,14 @@ static final class Tok implements Input.Tok {
int position,
int columnI,
boolean isToken,
int id) {
TokenKind kind) {
this.index = index;
this.originalText = originalText;
this.text = text;
this.position = position;
this.columnI = columnI;
this.isToken = isToken;
this.id = id;
this.kind = kind;
}

@Override
Expand Down Expand Up @@ -163,12 +174,8 @@ public String toString() {
.toString();
}

/**
* The token id used by the eclipse scanner. See {@link
* org.eclipse.jdt.core.compiler.ITerminalSymbols} for possible values.
*/
public int id() {
return id;
public TokenKind kind() {
return kind;
}
}

Expand Down Expand Up @@ -322,47 +329,54 @@ public ImmutableMap<Integer, Integer> getPositionToColumnMap() {

/** Lex the input and build the list of toks. */
private ImmutableList<Tok> buildToks(String text) throws FormatterException {
try {
ImmutableList<Tok> toks = buildToks(text, ImmutableSet.<Integer>of());
kN = getLast(toks).getIndex();
computeRanges(toks);
return toks;
} catch (InvalidInputException e) {
// jdt's scanner elects not to produce error messages, so we don't either
//
// problems will get caught (again!) and reported (with error messages!)
// during parsing
return ImmutableList.of();
}
ImmutableList<Tok> toks = buildToks(text, ImmutableSet.<TokenKind>of());
kN = getLast(toks).getIndex();
computeRanges(toks);
return toks;
}

/**
* Lex the input and build the list of toks.
*
* @param text the text to be lexed.
* @param stopIds a set of Eclipse token names which should cause lexing to stop. If one of these
* is found, the returned list will include tokens up to but not including that token.
* @param stopTokens a set of tokens which should cause lexing to stop. If one of these is found,
* the returned list will include tokens up to but not including that token.
*/
static ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds)
throws InvalidInputException, FormatterException {
stopIds =
ImmutableSet.<Integer>builder().addAll(stopIds).add(ITerminalSymbols.TokenNameEOF).build();
static ImmutableList<Tok> buildToks(String text, ImmutableSet<TokenKind> stopTokens)
throws FormatterException {
stopTokens = ImmutableSet.<TokenKind>builder().addAll(stopTokens).add(TokenKind.EOF).build();
Context context = new Context();
new JavacFileManager(context, true, UTF_8);
DiagnosticCollector<JavaFileObject> diagnosticCollector = new DiagnosticCollector<>();
context.put(DiagnosticListener.class, diagnosticCollector);
Log log = Log.instance(context);
log.useSource(
new SimpleJavaFileObject(URI.create("Source.java"), Kind.SOURCE) {
@Override
public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
return text;
}
});
DeferredDiagnosticHandler diagnostics = new DeferredDiagnosticHandler(log);
ImmutableList<RawTok> rawToks = JavacTokens.getTokens(text, context, stopTokens);
if (diagnostics.getDiagnostics().stream().anyMatch(d -> d.getKind() == Diagnostic.Kind.ERROR)) {
return ImmutableList.of(new Tok(0, "", "", 0, 0, true, null)); // EOF
}
int kN = 0;
IScanner scanner = ToolFactory.createScanner(true, true, true, "1.8");
scanner.setSource(text.toCharArray());
int textLength = text.length();
List<Tok> toks = new ArrayList<>();
int charI = 0;
int columnI = 0;
while (scanner.getCurrentTokenEndPosition() < textLength - 1) {
int tokenId = scanner.getNextToken();
if (stopIds.contains(tokenId)) {
for (RawTok t : rawToks) {
if (stopTokens.contains(t.kind())) {
break;
}
int charI0 = scanner.getCurrentTokenStartPosition();
int charI0 = t.pos();
// Get string, possibly with Unicode escapes.
String originalTokText = text.substring(charI0, scanner.getCurrentTokenEndPosition() + 1);
String tokText = new String(scanner.getCurrentTokenSource()); // Unicode escapes removed.
String originalTokText = text.substring(charI0, t.endPos());
String tokText =
t.kind() == TokenKind.STRINGLITERAL
? t.stringVal() // Unicode escapes removed.
: originalTokText;
char tokText0 = tokText.charAt(0); // The token's first character.
final boolean isToken; // Is this tok a token?
final boolean isNumbered; // Is this tok numbered? (tokens and comments)
Expand Down Expand Up @@ -427,7 +441,7 @@ static ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds)
charI,
columnI,
isToken,
tokenId));
t.kind()));
charI += originalTokText.length();
columnI = updateColumn(columnI, originalTokText);

Expand All @@ -437,18 +451,18 @@ static ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds)
"Unicode escapes not allowed in whitespace or multi-character operators");
}
for (String str : strings) {
toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, tokenId));
toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null));
charI += str.length();
columnI = updateColumn(columnI, originalTokText);
}
}
if (extraNewline != null) {
toks.add(new Tok(-1, extraNewline, extraNewline, charI, columnI, false, tokenId));
toks.add(new Tok(-1, extraNewline, extraNewline, charI, columnI, false, null));
columnI = 0;
charI += extraNewline.length();
}
}
toks.add(new Tok(kN, "", "", charI, columnI, true, ITerminalSymbols.TokenNameEOF)); // EOF tok.
toks.add(new Tok(kN, "", "", charI, columnI, true, null)); // EOF tok.
return ImmutableList.copyOf(toks);
}

Expand Down

0 comments on commit 946e01c

Please sign in to comment.