From fa5120ea121c3021c06ae1ea4cbc9bc9bd602879 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 29 Mar 2016 12:25:25 -0700 Subject: [PATCH] add context up the parse tree. Indent is part of alignment decision. The indent is a category that is relative to first token of some ancestor a delta up the tree. works great! --- java/codebuff.iml | 2 +- java/codebuff.ipr | 11 ++ java/pom.xml | 13 +- .../org/antlr/codebuff/CollectFeatures.java | 185 ++++++++++++++++-- java/src/org/antlr/codebuff/Formatter.java | 60 ++---- java/src/org/antlr/codebuff/Tool.java | 2 +- samples/small/LongStuffAligned.java | 3 +- 7 files changed, 203 insertions(+), 73 deletions(-) diff --git a/java/codebuff.iml b/java/codebuff.iml index ae954db..5fd7f68 100644 --- a/java/codebuff.iml +++ b/java/codebuff.iml @@ -14,7 +14,7 @@ - + diff --git a/java/codebuff.ipr b/java/codebuff.ipr index a5904bf..b3875d1 100644 --- a/java/codebuff.ipr +++ b/java/codebuff.ipr @@ -225,6 +225,17 @@ + + + + + + + + + + + diff --git a/java/pom.xml b/java/pom.xml index 4220a75..ac141d3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-runtime - 4.5.1 + 4.5.3-SNAPSHOT junit @@ -47,13 +47,6 @@ commons-lang3 3.4 - @@ -62,13 +55,13 @@ org.antlr antlr4-maven-plugin - 4.5.1 + 4.5.3-SNAPSHOT grammars - + org.antlr.codebuff.misc.RuleNodeWithAlt antlr diff --git a/java/src/org/antlr/codebuff/CollectFeatures.java b/java/src/org/antlr/codebuff/CollectFeatures.java index d94d117..ae9c540 100644 --- a/java/src/org/antlr/codebuff/CollectFeatures.java +++ b/java/src/org/antlr/codebuff/CollectFeatures.java @@ -6,6 +6,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -38,7 +39,21 @@ public class CollectFeatures { public static final int CAT_ALIGN_WITH_LIST_FIRST_ELEMENT = 3; public static final int CAT_ALIGN_WITH_PAIR = 4; - public static final int CAT_INDENT = 100; + /* We want to identify indentation from a parent's start token but that + parent could be a number of levels up the tree. The next category + values indicate indentation from the current token's left ancestor's + parent then it's parent and so on. For category value: + + CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN + i + + current token is indented from start token of node i levels up + from ancestor. + */ + public static final int CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN = 100; // left ancestor's first token is really current token + + public static final int CAT_INDENT = 200; + + // indexes into feature vector public static final int INDEX_PREV2_TYPE = 0; public static final int INDEX_PREV_TYPE = 1; @@ -52,12 +67,16 @@ public class CollectFeatures { public static final int INDEX_RULE = 9; // what rule are we in? public static final int INDEX_EARLIEST_RIGHT_ANCESTOR = 10; public static final int INDEX_EARLIEST_LEFT_ANCESTOR = 11; - public static final int INDEX_NEXT_TYPE = 12; - public static final int INDEX_INFO_FILE = 13; - public static final int INDEX_INFO_LINE = 14; - public static final int INDEX_INFO_CHARPOS = 15; + public static final int INDEX_ANCESTORS_PARENT4_RULE = 12; + public static final int INDEX_ANCESTORS_PARENT3_RULE = 13; + public static final int INDEX_ANCESTORS_PARENT2_RULE = 14; + public static final int INDEX_ANCESTORS_PARENT_RULE = 15; + public static final int INDEX_NEXT_TYPE = 16; + public static final int INDEX_INFO_FILE = 17; + public static final int INDEX_INFO_LINE = 18; + public static final int INDEX_INFO_CHARPOS = 19; - public static final int NUM_FEATURES = 15; + public static final int NUM_FEATURES = 20; public static FeatureMetaData[] FEATURES_INJECT_NL = { new FeatureMetaData(FeatureType.TOKEN, new String[] {"", "LT(-2)"}, 1), @@ -72,6 +91,10 @@ public class CollectFeatures { new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "rule"}, 2), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "right ancestor"}, 3), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "left ancestor"}, 3), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^4"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^3"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^2"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent"}, 1), FeatureMetaData.UNUSED, new FeatureMetaData(FeatureType.INFO_FILE, new String[] {"", "file"}, 0), new FeatureMetaData(FeatureType.INFO_LINE, new String[] {"", "line"}, 0), @@ -92,6 +115,10 @@ public class CollectFeatures { new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "rule"}, 2), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "right ancestor"}, 3), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "left ancestor"}, 3), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^4"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^3"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^2"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent"}, 1), new FeatureMetaData(FeatureType.TOKEN, new String[] {"", "LT(2)"}, 1), new FeatureMetaData(FeatureType.INFO_FILE, new String[] {"", "file"}, 0), new FeatureMetaData(FeatureType.INFO_LINE, new String[] {"", "line"}, 0), @@ -111,6 +138,10 @@ public class CollectFeatures { new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "rule"}, 2), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "right ancestor"}, 3), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "left ancestor"}, 3), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^4"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^3"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^2"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent"}, 1), new FeatureMetaData(FeatureType.TOKEN, new String[] {"", "LT(2)"}, 1), new FeatureMetaData(FeatureType.INFO_FILE, new String[] {"", "file"}, 0), new FeatureMetaData(FeatureType.INFO_LINE, new String[] {"", "line"}, 0), @@ -130,6 +161,10 @@ public class CollectFeatures { new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "rule"}, 2), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "right ancestor"}, 3), new FeatureMetaData(FeatureType.RULE, new String[] {"LT(1)", "left ancestor"}, 3), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^4"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^3"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent^2"}, 1), + new FeatureMetaData(FeatureType.RULE, new String[] {"ancestor's", "parent"}, 1), new FeatureMetaData(FeatureType.TOKEN, new String[] {"", "LT(2)"}, 1), new FeatureMetaData(FeatureType.INFO_FILE, new String[] {"", "file"}, 0), new FeatureMetaData(FeatureType.INFO_LINE, new String[] {"", "line"}, 0), @@ -257,8 +292,19 @@ else if ( earliestAncestorsParentStart!=null && { aligned = CAT_ALIGN_WITH_ANCESTORS_PARENT_FIRST_TOKEN; } - else if ( columnDelta>0 ) { - aligned = CAT_INDENT; // indent standard amount + else if ( columnDelta!=0 ) { + ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(parent, curToken); + ParserRuleContext ancestorParent = getParent(earliestLeftAncestor); + int indentedFromPos = curToken.getCharPositionInLine()-Formatter.INDENT_LEVEL; + ParserRuleContext indentParent = + earliestAncestorStartingAtCharPos(ancestorParent, indentedFromPos); + if ( indentParent!=null ) { + int deltaFromLeftAncestor = getDeltaToAncestor(earliestLeftAncestor, indentParent); + aligned = CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN+deltaFromLeftAncestor; + } + else { + aligned = CAT_INDENT; // indent standard amount + } } return aligned; @@ -405,6 +451,55 @@ public static ParserRuleContext earliestAncestorEndingWithToken(ParserRuleContex return prev; } + /** Walk upwards from node until we find p.start at char position and p.start + * is first token on a line; return null if there is no such ancestor p. + */ + public ParserRuleContext earliestAncestorStartingAtCharPos(ParserRuleContext node, int charpos) { + ParserRuleContext p = node; + while ( p!=null ) { + if ( isFirstOnLine(p.getStart()) && p.getStart().getCharPositionInLine()==charpos ) { + return p; + } + p = p.getParent(); + } + return null; + } + + /** Return the number of hops to get to ancestor from node or -1 if we + * don't find ancestor on path to root. + */ + public static int getDeltaToAncestor(ParserRuleContext node, ParserRuleContext ancestor) { + int n = 0; + ParserRuleContext p = node; + while ( p!=null && p!=ancestor ) { + n++; + p = p.getParent(); + } + if ( p==null ) return -1; + return n; + } + + public static ParserRuleContext getAncestor(ParserRuleContext node, int delta) { + System.out.print(node.getText()+" "+JavaParser.ruleNames[node.getRuleIndex()]+"+"+delta); + int n = 0; + ParserRuleContext p = node; + while ( p!=null && n!=delta ) { + n++; + p = p.getParent(); + } + System.out.println(" is "+JavaParser.ruleNames[p.getRuleIndex()]+":"+p.getAltNumber()); + return p; + } + + public boolean isFirstOnLine(Token t) { + tokens.seek(t.getTokenIndex()); // LT(1) + Token prevToken = tokens.LT(-1); + if ( prevToken==null ) { + return true; // if we are first token, must be first on line + } + return t.getLine()>prevToken.getLine(); + } + public static ParserRuleContext deepestCommonAncestor(ParserRuleContext t1, ParserRuleContext t2) { if ( t1==t2 ) return t1; List t1_ancestors = Trees.getAncestors(t1); @@ -443,29 +538,37 @@ public static int[] getNodeFeatures(Map tokenToNodeMap, TerminalNode prevTerminalNode = tokenToNodeMap.get(prevToken); ParserRuleContext parent = (ParserRuleContext)prevTerminalNode.getParent(); int prevTokenRuleIndex = parent.getRuleIndex(); + int prevTokenRuleAltNum = parent.getAltNumber(); ParserRuleContext prevEarliestRightAncestor = earliestAncestorEndingWithToken(parent, prevToken); int prevEarliestAncestorRuleIndex = -1; + int prevEarliestAncestorRuleAltNum = 0; int prevEarliestAncestorWidth = -1; if ( prevEarliestRightAncestor!=null ) { prevEarliestAncestorRuleIndex = prevEarliestRightAncestor.getRuleIndex(); + prevEarliestAncestorRuleAltNum = prevEarliestRightAncestor.getAltNumber(); prevEarliestAncestorWidth = prevEarliestRightAncestor.stop.getStopIndex()-prevEarliestRightAncestor.start.getStartIndex()+1; } // Get context information for current token parent = (ParserRuleContext)node.getParent(); int curTokensParentRuleIndex = parent.getRuleIndex(); + int curTokensParentRuleAltNumber = parent.getAltNumber(); ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(parent, curToken); int earliestAncestorWidth = -1; int earliestLeftAncestorRuleIndex = -1; + int earliestLeftAncestorRuleAlt = 0; if ( earliestLeftAncestor!=null ) { earliestLeftAncestorRuleIndex = earliestLeftAncestor.getRuleIndex(); + earliestLeftAncestorRuleAlt = earliestLeftAncestor.getAltNumber(); earliestAncestorWidth = earliestLeftAncestor.stop.getStopIndex()-earliestLeftAncestor.start.getStartIndex()+1; } ParserRuleContext earliestRightAncestor = earliestAncestorEndingWithToken(parent, curToken); int earliestRightAncestorRuleIndex = -1; + int earliestRightAncestorRuleAlt = 0; if ( earliestRightAncestor!=null ) { earliestRightAncestorRuleIndex = earliestRightAncestor.getRuleIndex(); + earliestRightAncestorRuleAlt = earliestRightAncestor.getAltNumber(); } int prevTokenEndCharPos = window.get(1).getCharPositionInLine() + window.get(1).getText().length(); @@ -479,22 +582,40 @@ public static int[] getNodeFeatures(Map tokenToNodeMap, // TODO: I don't think we can detect first element of list boolean startOfList = isFirstSiblingOfList(tokenToNodeMap, curToken); + // Get some context from parse tree + ParserRuleContext ancestorParent = null; + ParserRuleContext ancestorParent2 = null; + if ( earliestLeftAncestor==null ) { // just use regular parent then + ancestorParent = getParent(node); + ancestorParent2 = ancestorParent.getParent(); // get immediate parent for context + } + else { + ancestorParent = getParent(earliestLeftAncestor); // get parent but skip chain rules + ancestorParent2 = ancestorParent.getParent(); // get immediate parent for context + } + ParserRuleContext ancestorParent3 = ancestorParent2!=null ? ancestorParent2.getParent() : null; + ParserRuleContext ancestorParent4 = ancestorParent3!=null ? ancestorParent3.getParent() : null; + boolean curTokenStartsNewLine = window.get(2).getLine()>window.get(1).getLine(); int[] features = { window.get(0).getType(), - window.get(1).getType(), - prevTokenRuleIndex, + rulealt(prevTokenRuleIndex,prevTokenRuleAltNum), prevTokenEndCharPos, - prevEarliestAncestorRuleIndex, + rulealt(prevEarliestAncestorRuleIndex,prevEarliestAncestorRuleAltNum), window.get(2).getType(), // LT(1) startOfList ? 1 : 0, matchingSymbolOnDiffLine, curTokenStartsNewLine ? 1 : 0, - curTokensParentRuleIndex, - earliestRightAncestorRuleIndex, - earliestLeftAncestorRuleIndex, + rulealt(curTokensParentRuleIndex,curTokensParentRuleAltNumber), + rulealt(earliestRightAncestorRuleIndex,earliestRightAncestorRuleAlt), + rulealt(earliestLeftAncestorRuleIndex,earliestLeftAncestorRuleAlt), + ancestorParent4!=null ? rulealt(ancestorParent4.getRuleIndex(),ancestorParent4.getAltNumber()) : -1, + ancestorParent3!=null ? rulealt(ancestorParent3.getRuleIndex(),ancestorParent3.getAltNumber()) : -1, + ancestorParent2!=null ? rulealt(ancestorParent2.getRuleIndex(),ancestorParent2.getAltNumber()) : -1, + rulealt(ancestorParent.getRuleIndex(),ancestorParent.getAltNumber()), // always at least token's parent exists + window.get(3).getType(), // info @@ -657,7 +778,9 @@ public static String _toString(FeatureMetaData[] FEATURES, InputDocument doc, in break; case RULE : if ( features[i]>=0 ) { - String ruleName = ruleNames[features[i]]; + String ruleName = ruleNames[unrulealt(features[i])[0]]; + int ruleAltNum = unrulealt(features[i])[1]; + ruleName += ":"+ruleAltNum; abbrev = StringUtils.abbreviateMiddle(ruleName, "*", displayWidth); buf.append(String.format("%"+displayWidth+"s", abbrev)); } @@ -782,11 +905,30 @@ public static List getRealTokens(CommonTokenStream tokens) { return real; } + public static ParserRuleContext getParent(TerminalNode p) { + return getParent((ParserRuleContext)p.getParent()); + } + /** Same as p.getParent() except we scan through chain rule nodes */ public static ParserRuleContext getParent(ParserRuleContext p) { if ( p==null ) return null; ParserRuleContext lastValidParent = p.getParent(); - // now try to walk chain rules starting with the parent of the usual parent + if ( lastValidParent==null ) return null; // must have hit the root + + return parentClosure(p.getParent()); +// +// // now try to walk chain rules starting with the parent of the usual parent +// ParserRuleContext q = lastValidParent.getParent(); +// while ( q!=null && q.getChildCount()==1 ) { // while is a chain rule +// lastValidParent = q; +// q = q.getParent(); +// } +// return lastValidParent; + } + + // try to walk chain rules starting with the parent of the usual parent + public static ParserRuleContext parentClosure(ParserRuleContext p) { + ParserRuleContext lastValidParent = p; ParserRuleContext q = lastValidParent.getParent(); while ( q!=null && q.getChildCount()==1 ) { // while is a chain rule lastValidParent = q; @@ -795,4 +937,15 @@ public static ParserRuleContext getParent(ParserRuleContext p) { return lastValidParent; } + /** Pack a rule index and an alternative number into the same 32-bit integer. */ + public static int rulealt(int rule, int alt) { + if ( rule==-1 ) return -1; + return rule<<16 | alt; + } + + /** Return {rule index, rule alt number} */ + public static int[] unrulealt(int ra) { + if ( ra==-1 ) return new int[] {-1, ATN.INVALID_ALT_NUMBER}; + return new int[] {(ra>>16)&0xFFFF,ra&0xFFFF}; + } } diff --git a/java/src/org/antlr/codebuff/Formatter.java b/java/src/org/antlr/codebuff/Formatter.java index 57950ed..0ecb16a 100644 --- a/java/src/org/antlr/codebuff/Formatter.java +++ b/java/src/org/antlr/codebuff/Formatter.java @@ -17,6 +17,7 @@ import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_LIST_FIRST_ELEMENT; import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_PAIR; import static org.antlr.codebuff.CollectFeatures.CAT_INDENT; +import static org.antlr.codebuff.CollectFeatures.CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN; import static org.antlr.codebuff.CollectFeatures.CAT_NO_ALIGNMENT; import static org.antlr.codebuff.CollectFeatures.FEATURES_ALIGN; import static org.antlr.codebuff.CollectFeatures.FEATURES_INJECT_NL; @@ -25,6 +26,7 @@ import static org.antlr.codebuff.CollectFeatures.INDEX_PREV_END_COLUMN; import static org.antlr.codebuff.CollectFeatures.MAX_CONTEXT_DIFF_THRESHOLD; import static org.antlr.codebuff.CollectFeatures.earliestAncestorEndingWithToken; +import static org.antlr.codebuff.CollectFeatures.earliestAncestorStartingWithToken; import static org.antlr.codebuff.CollectFeatures.getListSiblings; import static org.antlr.codebuff.CollectFeatures.getMatchingLeftSymbol; import static org.antlr.codebuff.CollectFeatures.getNodeFeatures; @@ -34,6 +36,8 @@ import static org.antlr.codebuff.CollectFeatures.isAlignedWithFirstSiblingOfList; public class Formatter { + public static final int INDENT_LEVEL = 4; + protected final Corpus corpus; protected StringBuilder output = new StringBuilder(); protected InputDocument doc; @@ -125,12 +129,11 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) { features[INDEX_FIRST_ON_LINE] = injectNewline; // use \n prediction to match exemplars for alignment int align = alignClassifier.classify(k, features, corpus.align, MAX_CONTEXT_DIFF_THRESHOLD); - int indent = 0; //indentClassifier.classify(k, features, corpus.indent, CollectFeatures.MAX_CONTEXT_DIFF_THRESHOLD); int ws = wsClassifier.classify(k, features, corpus.injectWS, MAX_CONTEXT_DIFF_THRESHOLD); TokenPositionAnalysis tokenPositionAnalysis = - getTokenAnalysis(features, indexIntoRealTokens, tokenIndexInStream, injectNewline, align, indent, ws); + getTokenAnalysis(features, indexIntoRealTokens, tokenIndexInStream, injectNewline, align, ws); analysis.setSize(tokenIndexInStream+1); analysis.set(tokenIndexInStream, tokenPositionAnalysis); @@ -156,7 +159,7 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) { switch ( align ) { case CAT_INDENT : if ( firstTokenOnPrevLine!=null ) { // if not on first line, we can indent indent - int indentedCol = firstTokenOnPrevLine.getCharPositionInLine() + 4; + int indentedCol = firstTokenOnPrevLine.getCharPositionInLine() + INDENT_LEVEL; charPosInLine = indentedCol; output.append(Tool.spaces(indentedCol)); } @@ -197,10 +200,18 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) { break; case CAT_NO_ALIGNMENT : break; + + default : + if ( align>=CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN ) { + int deltaFromAncestor = align - CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN; + ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(parent, curToken); + ParserRuleContext ancestor = CollectFeatures.getAncestor(earliestLeftAncestor, deltaFromAncestor); + Token start = ancestor.getStart(); + int indentCol = start.getCharPositionInLine() + INDENT_LEVEL; + charPosInLine = indentCol; + output.append(Tool.spaces(indentCol)); + } } -// if ( currentIndent<0 ) currentIndent = 0; // don't allow bad indents to accumulate -// charPosInLine = currentIndent; -// output.append(Tool.spaces(currentIndent)); } else { // inject whitespace instead of \n? @@ -208,39 +219,6 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) { charPosInLine += ws; } -// if ( injectNewline>0 ) { -// output.append(Tool.newlines(injectNewline)); -// line++; -// TerminalNode node = tokenToNodeMap.get(tokens.get(tokenIndexInStream)); -// ParserRuleContext parent = (ParserRuleContext)node.getParent(); -// int myIndex = 0; -// ParserRuleContext earliestAncestor = CollectFeatures.earliestAncestorStartingWithToken(parent, curToken); -// if ( earliestAncestor!=null ) { -// ParserRuleContext commonAncestor = earliestAncestor.getParent(); -// List siblings = commonAncestor.getRuleContexts(earliestAncestor.getClass()); -// myIndex = siblings.indexOf(earliestAncestor); -// } -// if ( false ) { //if ( myIndex>0 && align>0 ) { // align with first sibling's start token -// ParserRuleContext commonAncestor = earliestAncestor.getParent(); -// List siblings = commonAncestor.getRuleContexts(earliestAncestor.getClass()); -// ParserRuleContext firstSibling = siblings.get(0); -// Token firstSiblingStartToken = firstSibling.getStart(); -// // align but don't update currentIndent -// charPosInLine = firstSiblingStartToken.getCharPositionInLine(); -// output.append(Tool.spaces(charPosInLine)); -// } -// else { -// currentIndent += indent; -// if ( currentIndent<0 ) currentIndent = 0; // don't allow bad indents to accumulate -// charPosInLine = currentIndent; -// output.append(Tool.spaces(currentIndent)); -// } -// } -// else { -// // inject whitespace instead of \n? -// output.append(Tool.spaces(ws)); -// charPosInLine += ws; -// } // update Token object with position information now that we are about // to emit it. curToken.setLine(line); @@ -257,7 +235,6 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) { public TokenPositionAnalysis getTokenAnalysis(int[] features, int indexIntoRealTokens, int tokenIndexInStream, int injectNewline, int alignWithPrevious, - int indent, int ws) { CommonToken curToken = (CommonToken)tokens.get(tokenIndexInStream); @@ -274,7 +251,6 @@ public TokenPositionAnalysis getTokenAnalysis(int[] features, int indexIntoRealT boolean prevIsWS = prevToken.getType()==JavaLexer.WS; int actualNL = Tool.count(prevToken.getText(), '\n'); int actualWS = Tool.count(prevToken.getText(), ' '); - int actualIndent = originalCurToken.getCharPositionInLine()-0;// currentIndent; boolean actualAlign = isAlignedWithFirstSiblingOfList(tokenToNodeMap, tokens, curToken); String newlinePredictionString = String.format("### line %d: predicted %d \\n actual %s", originalCurToken.getLine(), injectNewline, prevIsWS ? actualNL : "none"); @@ -282,8 +258,6 @@ public TokenPositionAnalysis getTokenAnalysis(int[] features, int indexIntoRealT originalCurToken.getLine(), alignWithPrevious==1?"align":"unaligned", actualAlign?"align":"unaligned"); - String indentPredictionString = String.format("### line %d: predicted indent %d actual %s", - originalCurToken.getLine(), indent, actualIndent); String wsPredictionString = String.format("### line %d: predicted %d ' ' actual %s", originalCurToken.getLine(), ws, prevIsWS ? actualWS : "none"); if ( failsafeTriggered ) { diff --git a/java/src/org/antlr/codebuff/Tool.java b/java/src/org/antlr/codebuff/Tool.java index 2df2dbe..b49c13b 100644 --- a/java/src/org/antlr/codebuff/Tool.java +++ b/java/src/org/antlr/codebuff/Tool.java @@ -297,7 +297,7 @@ public static void getFilenames_(File f, String inputFilePattern, List f // otherwise, if this is an input file, load it! else if ( inputFilePattern==null || f.getName().matches(inputFilePattern) ) { - files.add(f.getAbsolutePath()); + files.add(f.getAbsolutePath()); } } diff --git a/samples/small/LongStuffAligned.java b/samples/small/LongStuffAligned.java index fd9abd9..5cdcae8 100644 --- a/samples/small/LongStuffAligned.java +++ b/samples/small/LongStuffAligned.java @@ -5,8 +5,7 @@ public TokenPositionAnalysis getTokenAnalysis(int[] features, int injectNewline, int alignWithPrevious, int indent, - int ws) - { + int ws) { CommonToken curToken = foo; CommonToken prevToken = 1; if ( ws==0 && cannotJoin(realTokens.get(indexIntoRealTokens-1), curToken) ) {