Skip to content

Commit

Permalink
Re-write formatting of ranges
Browse files Browse the repository at this point in the history 
The new approach uses token ranges instead of line ranges internally. The
formatter records the token indices of boundaries where it's safe to start/end
partial formatting, and then reformatting is only done for entire regions.

Regions that can be re-formatted atomically include:
-package declarations
-import declarations
-type declaration signatures (e.g. 'class' up to the first '{')
-method declaration signatures (e.g. modifiers up to '{')
-field declarations
-entire (non-block) statements
-lines containing only trailing braces
-------------
Created by MOE: http://code.google.com/p/moe-java
MOE_MIGRATED_REVID=93730338
  • Loading branch information
@cushon
cushon committed May 23, 2015
1 parent 40c5a74 commit 9d330a9
Show file tree
Hide file tree
Showing 11 changed files with 712 additions and 424 deletions.
3 changes: 3 additions & 0 deletions core/src/main/java/com/google/googlejavaformat/FakeOutput.java
View file
Expand Up @@ -82,6 +82,9 @@ public CommentsHelper getCommentsHelper() {
@Override
public void blankLine(int k, boolean wanted) {}

@Override
public void markForPartialFormat(int k) {}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/com/google/googlejavaformat/InputOutput.java
View file
Expand Up @@ -101,7 +101,7 @@ public static Map<Integer, Range<Integer>> makeKToIJ(InputOutput put, int kN) {
Map<Integer, Range<Integer>> map = new HashMap<>();
int ijN = put.getLineCount();
LOOP:
for (int ij = 0; ij < ijN; ij++) {
for (int ij = 0; ij <= ijN; ij++) {
Range<Integer> range = put.getRanges(ij).canonical(INTEGERS);
for (int k = range.lowerEndpoint(); k < range.upperEndpoint(); k++) {
if (map.containsKey(k)) {
Expand Down
9 changes: 9 additions & 0 deletions core/src/main/java/com/google/googlejavaformat/OpsBuilder.java
View file
Expand Up @@ -233,6 +233,15 @@ public final void breakOp(
ops.add(Doc.Break.make(fillMode, flat, plusIndent, optionalTag));
}

/**
* Make the boundary of a region that can be partially formatted. The
* boundary will be included in the following region, e.g.:
* [[boundary0, boundary1), [boundary1, boundary2), ...].
*/
public void markForPartialFormat() {
output.markForPartialFormat(getI(input.getTokens().get(tokenI)));
}

/**
* Add a list of {@link Op}s.
* @param newOps the list of {@link Op}s
Expand Down
8 changes: 7 additions & 1 deletion core/src/main/java/com/google/googlejavaformat/Output.java
View file
Expand Up @@ -27,7 +27,7 @@ public abstract class Output extends InputOutput {
* Unique identifier for a break.
*/
public static final class BreakTag {}

/**
* Indent by outputting {@code indent} spaces.
* @param indent the current indent
Expand Down Expand Up @@ -67,6 +67,12 @@ public static final class BreakTag {}
*/
public abstract void blankLine(int k, boolean wanted);

/**
* Marks the boundary of a region that can be partially formatted.
* @param k the token index
*/
public abstract void markForPartialFormat(int k);

/**
* Get the {@link CommentsHelper}.
* @return the {@link CommentsHelper}
Expand Down
107 changes: 8 additions & 99 deletions core/src/main/java/com/google/googlejavaformat/java/Formatter.java
View file
Expand Up @@ -15,14 +15,11 @@
package com.google.googlejavaformat.java;

import com.google.common.collect.DiscreteDomain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
import com.google.googlejavaformat.Doc;
import com.google.googlejavaformat.DocBuilder;
import com.google.googlejavaformat.InputOutput;
import com.google.googlejavaformat.Op;
import com.google.googlejavaformat.OpsBuilder;

Expand Down Expand Up @@ -116,7 +113,7 @@ public String formatSource(String input) throws FormatterException {
RangeSet<Integer> lineRangeSet = TreeRangeSet.create();
lineRangeSet.add(Range.<Integer>all());
try {
javaOutput.writeMerged(result, lineRangeSet, MAX_WIDTH, errors);
javaOutput.writeMerged(result, lineRangeSet);
} catch (IOException ignored) {
throw new AssertionError("IOException impossible for StringWriter");
}
Expand All @@ -141,113 +138,25 @@ public String formatSource(String input, List<Range<Integer>> characterRanges)
throw new FormatterException(errors.get(0));
}
StringBuilder result = new StringBuilder(input.length());
RangeSet<Integer> lineRangeSet = characterRangesToLineRangeSet(javaInput, characterRanges);
RangeSet<Integer> tokenRangeSet = characterRangesToTokenRanges(javaInput, characterRanges);
try {
javaOutput.writeMerged(result, lineRangeSet, MAX_WIDTH, errors);
javaOutput.writeMerged(result, tokenRangeSet);
} catch (IOException ignored) {
throw new AssertionError("IOException impossible for StringWriter");
}
return result.toString();
}

/**
* Emit a list of {@link Replacement}s to convert from input to output.
* @param input the input compilation unit
* @param characterRanges the character ranges to reformat
* @return a list of {@link Replacement}s, reverse-sorted from high index to low index, without
* overlaps
* @throws FormatterException if the input string cannot be parsed
*/
public ImmutableList<Replacement> getFormatReplacements(
String input, List<Range<Integer>> characterRanges) throws FormatterException {
List<Replacement> replacements = new ArrayList<>();
JavaInput javaInput = new JavaInput(input);
JavaOutput javaOutput = new JavaOutput(javaInput, new JavaCommentsHelper(), false);
List<String> errors = new ArrayList<>();
format(javaInput, javaOutput, MAX_WIDTH, errors, 1);
if (!errors.isEmpty()) {
throw new FormatterException(errors.get(0));
}
RangeSet<Integer> inputLineRangeSet = characterRangesToLineRangeSet(javaInput, characterRanges);
List<JavaOutput.RunInfo> lineInfos = javaOutput.pickRuns(inputLineRangeSet);
/*
* There is one {@code lineInfo} for each line of the merged output: an unformatted
* {@link From#INPUT} line or a formatted {@link From#OUTPUT} line. Each {@link Replacement}
* represents a run of {@link From#OUTPUT}s; it specifies the character range of the replaced
* input lines and the {@code String} value of the replacing output lines. The
* {@link Replacement}s do not overlap, since the lines do not.
*/
JavaOutput.From trackingFrom = JavaOutput.From.OUTPUT; // Which are we currently tracking?
int inputLineI = 0; // Up to this line in the unformatted input, during From.INPUT.
int inputCharacterI = 0; // Up to this character in the unformatted input, during From.INPUT.
int outputLineJ = 0; // Up to this line in the formatted output, during From.OUTPUT.
int outputLineJ0 = 0; // When did we switch to output?
for (int ij = 0; ij < lineInfos.size(); ij++) {
JavaOutput.RunInfo lineInfo = lineInfos.get(ij);
if (trackingFrom == JavaOutput.From.INPUT) {
if (lineInfo.from == JavaOutput.From.INPUT) {
// Continue tracking input.
while (inputLineI <= lineInfo.ij0) {
inputCharacterI += javaInput.getLine(inputLineI++).length() + 1; // Include '\n'.
}
} else {
// Done tracking input; switch to output.
trackingFrom = JavaOutput.From.OUTPUT;
outputLineJ0 = lineInfo.ij0;
outputLineJ = lineInfo.ij0 + 1;
}
} else {
if (lineInfo.from == JavaOutput.From.INPUT) {
// Done tracking output; switch to input and emit a Replacement.
int inputRange0 = inputCharacterI; // Where we switched to output.
trackingFrom = JavaOutput.From.INPUT;
while (inputLineI < lineInfo.ij0) {
inputCharacterI += javaInput.getLine(inputLineI++).length() + 1; // Include '\n'.
}
int inputRange1 = inputCharacterI;
inputCharacterI += javaInput.getLine(inputLineI++).length() + 1; // Include '\n'.
Range<Integer> range = Range.closedOpen(inputRange0, inputRange1);
if (!range.isEmpty() || outputLineJ0 < outputLineJ) {
// Emit a Replacement.
replacements.add(
Replacement.create(range, getLines(javaOutput, outputLineJ0, outputLineJ)));
}
} else {
// Continue tracking output.
outputLineJ = lineInfo.ij0 + 1;
}
}
}
if (trackingFrom == JavaOutput.From.OUTPUT && outputLineJ0 < outputLineJ) {
// Emit final Replacement.
Range<Integer> range = Range.closedOpen(inputCharacterI, javaInput.getText().length());
if (!range.isEmpty() || outputLineJ0 < outputLineJ) {
// Emit a Replacement.
replacements.add(
Replacement.create(range, getLines(javaOutput, outputLineJ0, outputLineJ)));
}
}
return ImmutableList.copyOf(Lists.reverse(replacements));
}

private static RangeSet<Integer> characterRangesToLineRangeSet(
private static RangeSet<Integer> characterRangesToTokenRanges(
JavaInput javaInput, List<Range<Integer>> characterRanges) throws FormatterException {
RangeSet<Integer> lineRangeSet = TreeRangeSet.create();
RangeSet<Integer> tokenRangeSet = TreeRangeSet.create();
for (Range<Integer> characterRange0 : characterRanges) {
Range<Integer> characterRange = characterRange0.canonical(DiscreteDomain.integers());
lineRangeSet.add(
javaInput.characterRangeToLineRange(
tokenRangeSet.add(
javaInput.characterRangeToTokenRange(
characterRange.lowerEndpoint(),
characterRange.upperEndpoint() - characterRange.lowerEndpoint()));
}
return lineRangeSet.subRangeSet(Range.openClosed(0, javaInput.getLineCount()));
}

private static String getLines(InputOutput put, int ij0, int ij1) {
StringBuilder builder = new StringBuilder();
for (int ij = ij0; ij < ij1; ij++) {
builder.append(put.getLine(ij)).append('\n');
}
return builder.toString();
return tokenRangeSet;
}
}
71 changes: 54 additions & 17 deletions core/src/main/java/com/google/googlejavaformat/java/JavaInput.java
View file
Expand Up @@ -14,6 +14,8 @@

package com.google.googlejavaformat.java;

import static com.google.common.base.MoreObjects.firstNonNull;

import com.google.common.base.MoreObjects;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -215,6 +217,9 @@ public String toString() {
private final ImmutableList<Token> tokens; // The Tokens for this input.
private final ImmutableSortedMap<Integer, Token> positionTokenMap; // Map position to Token.

/** Map from Tok index to the associated Token. */
private final Token[] kToToken;

/**
* Input constructor.
* @param text the input text
Expand All @@ -228,12 +233,29 @@ public JavaInput(String text) throws FormatterException {
ImmutableList<Tok> toks = buildToks(text, chars);
positionToColumnMap = makePositionToColumnMap(toks);
tokens = buildTokens(toks);
ImmutableSortedMap.Builder<Integer, Token> locationTokenMap =
ImmutableSortedMap.naturalOrder();
ImmutableSortedMap.Builder<Integer, Token> locationTokenMap = ImmutableSortedMap.naturalOrder();
for (Token token : tokens) {
locationTokenMap.put(token.getTok().getPosition(), token);
locationTokenMap.put(JavaOutput.startTok(token).getPosition(), token);
}
positionTokenMap = locationTokenMap.build();

// adjust kN for EOF
kToToken = new Token[kN + 1];
for (Token token : tokens) {
for (Input.Tok tok : token.getToksBefore()) {
if (tok.getIndex() < 0) {
continue;
}
kToToken[tok.getIndex()] = token;
}
kToToken[token.getTok().getIndex()] = token;
for (Input.Tok tok : token.getToksAfter()) {
if (tok.getIndex() < 0) {
continue;
}
kToToken[tok.getIndex()] = token;
}
}
}

private static ImmutableMap<Integer, Integer> makePositionToColumnMap(List<Tok> toks) {
Expand Down Expand Up @@ -368,15 +390,15 @@ private static ImmutableList<Token> buildTokens(List<Tok> toks) {
int k = 0;
int kN = toks.size();

while (k < kN) {
while (k < kN) {
// Remaining non-tokens before the token go here.
ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder();

while (!toks.get(k).isToken()) {
toksBefore.add(toks.get(k++));
}
Tok tok = toks.get(k++);

// Non-tokens starting on the same line go here too.
ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder();
while (k < kN && !"\n".equals(toks.get(k).getText()) && !toks.get(k).isToken()) {
Expand All @@ -386,8 +408,7 @@ private static ImmutableList<Token> buildTokens(List<Tok> toks) {
break;
}
}
tokens.add(
new Token(toksBefore.build(), tok, toksAfter.build()));
tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
}
return tokens.build();
}
Expand Down Expand Up @@ -439,30 +460,38 @@ int getLineNumberHi(Token token) {
}

/**
* Convert from an offset and length flag pair to a range of line numbers.
* Convert from an offset and length flag pair to a token range.
* @param offset the {@code 0}-based offset in characters
* @param length the length in characters
* @return the {@code 0}-based {@link Range} of line numbers
* @throws FormatterException
* @return the {@code 0}-based {@link Range} of tokens
* @throws FormatterException
*/
Range<Integer> characterRangeToLineRange(int offset, int length) throws FormatterException {
Range<Integer> characterRangeToTokenRange(int offset, int length) throws FormatterException {
int requiredLength = offset + length;
if (requiredLength > text.length()) {
throw new FormatterException(
String.format(
"invalid length %d, offset + length (%d) is outside the file",
"invalid length %d, offset + length (%d) is outside the file",
requiredLength,
requiredLength));
}
if (length <= 0) {
return Formatter.EMPTY_RANGE;
}
NavigableMap<Integer, JavaInput.Token> map = getPositionTokenMap();
Map.Entry<Integer, JavaInput.Token> tokenEntryLo = map.floorEntry(offset);
Map.Entry<Integer, JavaInput.Token> tokenEntryHi = map.ceilingEntry(offset + length - 1);
Map.Entry<Integer, JavaInput.Token> tokenEntryLo =
firstNonNull(map.floorEntry(offset), map.firstEntry());
Map.Entry<Integer, JavaInput.Token> tokenEntryHi =
firstNonNull(map.ceilingEntry(offset + length - 1), map.lastEntry());
return Range.closedOpen(
tokenEntryLo.getValue().getTok().getIndex(),
tokenEntryHi.getValue().getTok().getIndex() + 1);
}

Range <Integer> lineRangeToTokenRange(Range<Integer> lineRange) {
return Range.closedOpen(
tokenEntryLo == null ? 0 : getLineNumberLo(tokenEntryLo.getValue()),
tokenEntryHi == null ? getLineCount() : getLineNumberHi(tokenEntryHi.getValue()) + 1);
getRange0s(lineRange.lowerEndpoint()).lowerEndpoint(),
getRange1s(lineRange.upperEndpoint() - 1).upperEndpoint());
}

/**
Expand All @@ -473,6 +502,14 @@ int getkN() {
return kN;
}

/**
* Get the Token by index.
* @param k the token index
*/
Token getToken(int k) {
return kToToken[k];
}

/**
* Get the input tokens.
* @return the input tokens
Expand Down
8 changes: 8 additions & 0 deletions core/src/main/java/com/google/googlejavaformat/java/JavaInputAstVisitor.java
View file
Expand Up @@ -292,6 +292,7 @@ public JavaInputAstVisitor(OpsBuilder builder, int indentMultiplier) {
public boolean visit(CompilationUnit node) {
boolean first = true;
if (node.getPackage() != null) {
builder.markForPartialFormat();
visit(node.getPackage());
builder.breakOp();
first = false;
Expand All @@ -301,6 +302,7 @@ public boolean visit(CompilationUnit node) {
builder.blankLineWanted(true);
}
for (ImportDeclaration importDeclaration : (List<ImportDeclaration>) node.imports()) {
builder.markForPartialFormat();
visit(importDeclaration);
builder.breakOp();
}
Expand All @@ -310,10 +312,13 @@ public boolean visit(CompilationUnit node) {
if (!first) {
builder.blankLineWanted(true);
}
builder.markForPartialFormat();
type.accept(this);
builder.breakOp();
first = false;
}
// set a partial format marker at EOF to make sure we can format the entire file
builder.markForPartialFormat();
return false;
}

Expand Down Expand Up @@ -2024,6 +2029,7 @@ private void visitBlock(
tokenBreakTrailingComment("{", plusTwo);
for (Statement statement : (List<Statement>) node.statements()) {
builder.forcedBreak();
builder.markForPartialFormat();
statement.accept(this);
}
builder.close();
Expand All @@ -2032,6 +2038,7 @@ private void visitBlock(
if (allowTrailingBlankLine == AllowTrailingBlankLine.NO) {
builder.blankLineWanted(false);
}
builder.markForPartialFormat();
token("}", plusTwo);
}
}
Expand Down Expand Up @@ -2947,6 +2954,7 @@ void addBodyDeclarations(
if (!first && (thisOneGetsBlankLineBefore || lastOneGotBlankLineBefore)) {
builder.blankLineWanted(true);
}
builder.markForPartialFormat();
bodyDeclaration.accept(this);
first = false;
lastOneGotBlankLineBefore = thisOneGetsBlankLineBefore;
Expand Down

0 comments on commit 9d330a9

Please sign in to comment.