rm start of list feature, find alignment with rule delta, child index…

java/codebuff.ipr

@@ -37,6 +37,24 @@
   <component name="EntryPointsManager">
     <entry_points version="2.0" />
   </component>
+  <component name="InspectionProjectProfileManager">
+    <profile version="1.0">
+      <option name="myName" value="Project Default" />
+      <option name="myLocal" value="true" />
+      <inspection_tool class="FieldCanBeLocal" enabled="false" level="WARNING" enabled_by_default="false" />
+      <inspection_tool class="FinalPrivateMethod" enabled="false" level="WARNING" enabled_by_default="false" />
+      <inspection_tool class="FinalStaticMethod" enabled="false" level="WARNING" enabled_by_default="false" />
+      <inspection_tool class="LoggerInitializedWithForeignClass" enabled="false" level="WARNING" enabled_by_default="false">
+        <option name="loggerClassName" value="org.apache.log4j.Logger,org.slf4j.LoggerFactory,org.apache.commons.logging.LogFactory,java.util.logging.Logger" />
+        <option name="loggerFactoryMethodName" value="getLogger,getLogger,getLog,getLogger" />
+      </inspection_tool>
+      <inspection_tool class="ParameterCanBeLocal" enabled="false" level="WARNING" enabled_by_default="false" />
+      <inspection_tool class="TypeParameterHidesVisibleType" enabled="false" level="WARNING" enabled_by_default="false" />
+    </profile>
+    <option name="PROJECT_PROFILE" value="Project Default" />
+    <option name="USE_PROJECT_PROFILE" value="true" />
+    <version value="1.0" />
+  </component>
   <component name="MavenProjectsManager">
     <option name="originalFiles">
       <list>

java/src/org/antlr/codebuff/Corpus.java

@@ -86,7 +86,9 @@ public void buildTokenContextIndex() {
 		for (int i=0; i<X.size(); i++) {
 			int curTokenRuleIndex = X.get(i)[CollectFeatures.INDEX_RULE];
 			int prevTokenRuleIndex = X.get(i)[CollectFeatures.INDEX_PREV_RULE];
-			Pair<Integer, Integer> key = new Pair<>(prevTokenRuleIndex, curTokenRuleIndex);
+			int pr = CollectFeatures.unrulealt(prevTokenRuleIndex)[0];
+			int cr = CollectFeatures.unrulealt(curTokenRuleIndex)[0];
+			Pair<Integer, Integer> key = new Pair<>(pr, cr);
 			List<Integer> vectorIndexes = curAndPrevTokenRuleIndexToVectorsMap.get(key);
 			if ( vectorIndexes==null ) {
 				vectorIndexes = new ArrayList<>();

java/src/org/antlr/codebuff/Formatter.java

@@ -6,16 +6,14 @@
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.WritableToken;
 import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
 import java.util.List;
 import java.util.Map;
 import java.util.Vector;
 
-import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_ANCESTORS_PARENT_FIRST_TOKEN;
-import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_ANCESTOR_FIRST_TOKEN;
-import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_LIST_FIRST_ELEMENT;
-import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_PAIR;
+import static org.antlr.codebuff.CollectFeatures.CAT_ALIGN_WITH_ANCESTOR_CHILD;
 import static org.antlr.codebuff.CollectFeatures.CAT_INDENT;
 import static org.antlr.codebuff.CollectFeatures.CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN;
 import static org.antlr.codebuff.CollectFeatures.CAT_NO_ALIGNMENT;
@@ -25,15 +23,11 @@
 import static org.antlr.codebuff.CollectFeatures.INDEX_FIRST_ON_LINE;
 import static org.antlr.codebuff.CollectFeatures.INDEX_PREV_END_COLUMN;
 import static org.antlr.codebuff.CollectFeatures.MAX_CONTEXT_DIFF_THRESHOLD;
-import static org.antlr.codebuff.CollectFeatures.earliestAncestorEndingWithToken;
 import static org.antlr.codebuff.CollectFeatures.earliestAncestorStartingWithToken;
-import static org.antlr.codebuff.CollectFeatures.getListSiblings;
-import static org.antlr.codebuff.CollectFeatures.getMatchingLeftSymbol;
 import static org.antlr.codebuff.CollectFeatures.getNodeFeatures;
 import static org.antlr.codebuff.CollectFeatures.getRealTokens;
 import static org.antlr.codebuff.CollectFeatures.getTokensOnPreviousLine;
 import static org.antlr.codebuff.CollectFeatures.indexTree;
-import static org.antlr.codebuff.CollectFeatures.isAlignedWithFirstSiblingOfList;
 
 public class Formatter {
 	public static final int INDENT_LEVEL = 4;
@@ -75,6 +69,7 @@ public Formatter(Corpus corpus, InputDocument doc, int tabSize) {
 		wsClassifier = new CodekNNClassifier(corpus, FEATURES_INJECT_WS);
 		alignClassifier = new CodekNNClassifier(corpus, FEATURES_ALIGN);
 //		k = (int)Math.sqrt(corpus.X.size());
+//		k = 7;
 		k = 11;
 		this.tabSize = tabSize;
 	}
@@ -154,7 +149,6 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) {
 
 			TerminalNode node = tokenToNodeMap.get(curToken);
 			ParserRuleContext parent = (ParserRuleContext)node.getParent();
-			ParserRuleContext earliestRightAncestor = earliestAncestorEndingWithToken(parent, curToken);
 
 			switch ( align ) {
 				case CAT_INDENT :
@@ -164,47 +158,43 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) {
 						output.append(Tool.spaces(indentedCol));
 					}
 					break;
-				case CAT_ALIGN_WITH_ANCESTOR_FIRST_TOKEN :
-					if ( earliestRightAncestor!=null ) {
-						Token earliestRightAncestorStart = earliestRightAncestor.getStart();
-						int linedUpCol = earliestRightAncestorStart.getCharPositionInLine();
-						charPosInLine = linedUpCol;
-						output.append(Tool.spaces(linedUpCol));
-					}
-					break;
-				case CAT_ALIGN_WITH_ANCESTORS_PARENT_FIRST_TOKEN :
-					if ( earliestRightAncestor!=null ) {
-						ParserRuleContext earliestAncestorParent = earliestRightAncestor.getParent();
-						if ( earliestAncestorParent!=null ) {
-							Token earliestAncestorParentStart = earliestAncestorParent.getStart();
-							int linedUpCol = earliestAncestorParentStart.getCharPositionInLine();
-							charPosInLine = linedUpCol;
-							output.append(Tool.spaces(linedUpCol));
-						}
-					}
-					break;
-				case CAT_ALIGN_WITH_LIST_FIRST_ELEMENT :
-					List<ParserRuleContext> listSiblings = getListSiblings(tokenToNodeMap, curToken);
-					if ( listSiblings!=null ) {
-						ParserRuleContext firstSibling = listSiblings.get(0);
-						int linedUpCol = firstSibling.getStart().getCharPositionInLine();
-						charPosInLine = linedUpCol;
-						output.append(Tool.spaces(linedUpCol));
-					}
-					break;
-				case CAT_ALIGN_WITH_PAIR :
-					TerminalNode matchingLeftSymbol = getMatchingLeftSymbol(doc, node);
-					int linedUpCol = matchingLeftSymbol.getSymbol().getCharPositionInLine();
-					charPosInLine = linedUpCol;
-					output.append(Tool.spaces(linedUpCol));
-					break;
 				case CAT_NO_ALIGNMENT :
 					break;
 
 				default :
-					if ( align>=CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN ) {
-						int deltaFromAncestor = align - CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN;
+					if ( (align&0xFF)==CAT_ALIGN_WITH_ANCESTOR_CHILD ) {
+						int[] deltaChild = CollectFeatures.unaligncat(align);
+						int deltaFromAncestor = deltaChild[0];
+						int childIndex = deltaChild[1];
 						ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(parent, curToken);
+						if ( earliestLeftAncestor==null ) {
+							earliestLeftAncestor = parent;
+						}
+						ParserRuleContext ancestor = CollectFeatures.getAncestor(earliestLeftAncestor, deltaFromAncestor);
+						ParseTree child = ancestor.getChild(childIndex);
+						Token start = null;
+						if ( child instanceof ParserRuleContext ) {
+							start = ((ParserRuleContext) child).getStart();
+						}
+						else if ( child instanceof TerminalNode ){
+							start = ((TerminalNode)child).getSymbol();
+						}
+						else {
+							// uh oh.
+							System.err.println("Whoops. Tried access invalid child");
+						}
+						if ( start!=null ) {
+							int indentCol = start.getCharPositionInLine();
+							charPosInLine = indentCol;
+							output.append(Tool.spaces(indentCol));
+						}
+					}
+					else if ( (align&0xFF)==CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN ) {
+						int deltaFromAncestor = CollectFeatures.unindentcat(align);
+						ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(parent, curToken);
+						if ( earliestLeftAncestor==null ) {
+							earliestLeftAncestor = parent;
+						}
 						ParserRuleContext ancestor = CollectFeatures.getAncestor(earliestLeftAncestor, deltaFromAncestor);
 						Token start = ancestor.getStart();
 						int indentCol = start.getCharPositionInLine() + INDENT_LEVEL;
@@ -251,13 +241,12 @@ public TokenPositionAnalysis getTokenAnalysis(int[] features, int indexIntoRealT
 		boolean prevIsWS = prevToken.getType()==JavaLexer.WS;
 		int actualNL = Tool.count(prevToken.getText(), '\n');
 		int actualWS = Tool.count(prevToken.getText(), ' ');
-		boolean actualAlign = isAlignedWithFirstSiblingOfList(tokenToNodeMap, tokens, curToken);
 		String newlinePredictionString = String.format("### line %d: predicted %d \\n actual %s",
 		                                               originalCurToken.getLine(), injectNewline, prevIsWS ? actualNL : "none");
 		String alignPredictionString = String.format("### line %d: predicted %s actual %s",
 		                                             originalCurToken.getLine(),
 		                                             alignWithPrevious==1?"align":"unaligned",
-		                                             actualAlign?"align":"unaligned");
+		                                             "?");
 		String wsPredictionString = String.format("### line %d: predicted %d ' ' actual %s",
 		                                          originalCurToken.getLine(), ws, prevIsWS ? actualWS : "none");
 		if ( failsafeTriggered ) {

java/src/org/antlr/codebuff/InputDocument.java

@@ -41,7 +41,10 @@ public String getLine(int line) {
 		if ( lines==null ) {
 			lines = Arrays.asList(content.split("\n"));
 		}
-		return lines.get(line-1);
+		if ( line>0 ) {
+			return lines.get(line-1);
+		}
+		return null;
 	}
 
 	public double getIncorrectWSRate() {

java/src/org/antlr/codebuff/Neighbor.java

@@ -21,7 +21,12 @@ public String toString(FeatureMetaData[] FEATURES, List<Integer> Y) {
 		String lineText = doc.getLine(line);
 		int col = X[CollectFeatures.INDEX_INFO_CHARPOS];
 		// insert a dot right before char position
-		lineText = lineText.substring(0,col) + '\u00B7' + lineText.substring(col,lineText.length());
-		return String.format("%s (cat=%d,d=%1.3f): %s", features, Y.get(corpusVectorIndex), distance, lineText);
+		if ( lineText!=null ) {
+			lineText = lineText.substring(0, col)+'\u00B7'+lineText.substring(col, lineText.length());
+		}
+		int cat = Y.get(corpusVectorIndex);
+		int[] elements = CollectFeatures.unaligncat(cat);
+		String display = String.format("%d|%d|%d", cat&0xFF, elements[0], elements[1]);
+		return String.format("%s (cat=%s,d=%1.3f): %s", features, display, distance, lineText);
 	}
 }

java/src/org/antlr/codebuff/TokenPositionAnalysis.java

@@ -7,9 +7,6 @@ public class TokenPositionAnalysis {
 	public String ws = "n/a";
 	public String align = "n/a";
 
-	public TokenPositionAnalysis() {
-	}
-
 	public TokenPositionAnalysis(String newline, String align, String ws) {
 		this.align = align;
 		this.newline = newline;

java/src/org/antlr/codebuff/Tool.java

@@ -14,10 +14,7 @@
 import org.antlr.v4.runtime.misc.Pair;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.nio.file.FileSystems;
@@ -119,21 +116,6 @@ public static Corpus train(String rootDir,
 		return corpus;
 	}
 
-	public void saveCSV(List<InputDocument> documents, String dir) throws IOException {
-		FileWriter fw = new FileWriter(dir+"/style.csv");
-		BufferedWriter bw = new BufferedWriter(fw);
-//		bw.write(Utils.join(CollectFeatures.FEATURE_NAMES, ", "));
-		bw.write("\n");
-		for (InputDocument doc : documents) {
-			for (int[] record : doc.featureVectors) {
-				String r = join(record, ", ");
-				bw.write(r);
-				bw.write('\n');
-			}
-		}
-		bw.close();
-	}
-
 	public static Corpus processSampleDocs(List<InputDocument> docs,
 										   Class<? extends Lexer> lexerClass,
 										   Class<? extends Parser> parserClass,
@@ -164,9 +146,7 @@ public static Corpus processSampleDocs(List<InputDocument> docs,
 	}
 
 	/** Parse document, save feature vectors to the doc but return it also */
-	public static void process(InputDocument doc, int tabSize, Map<String, List<Pair<Integer, Integer>>> ruleToPairsBag)
-		throws Exception
-	{
+	public static void process(InputDocument doc, int tabSize, Map<String, List<Pair<Integer, Integer>>> ruleToPairsBag) {
 		CollectFeatures collector = new CollectFeatures(doc, tabSize, ruleToPairsBag);
 		collector.computeFeatureVectors();
 
@@ -234,7 +214,7 @@ public static List<InputDocument> load(List<String> fileNames,
 										   int tabSize)
 		throws Exception
 	{
-		List<InputDocument> input = new ArrayList<InputDocument>(fileNames.size());
+		List<InputDocument> input = new ArrayList<>(fileNames.size());
 		int i = 0;
 		for (String f : fileNames) {
 			InputDocument doc = load(f, lexerClass, tabSize);
@@ -281,17 +261,17 @@ public static InputDocument load(String fileName,
 	}
 
 	public static List<String> getFilenames(File f, String inputFilePattern) throws Exception {
-		List<String> files = new ArrayList<String>();
+		List<String> files = new ArrayList<>();
 		getFilenames_(f, inputFilePattern, files);
 		return files;
 	}
 
-	public static void getFilenames_(File f, String inputFilePattern, List<String> files) throws Exception {
+	public static void getFilenames_(File f, String inputFilePattern, List<String> files) {
 		// If this is a directory, walk each file/dir in that directory
 		if (f.isDirectory()) {
 			String flist[] = f.list();
-			for (int i=0; i < flist.length; i++) {
-				getFilenames_(new File(f, flist[i]), inputFilePattern, files);
+			for (String aFlist : flist) {
+				getFilenames_(new File(f, aFlist), inputFilePattern, files);
 			}
 		}
 
@@ -338,7 +318,6 @@ public static List<CommonToken> copy(CommonTokenStream tokens) {
 		List<CommonToken> copy = new ArrayList<>();
 		tokens.fill();
 		for (Token t : tokens.getTokens()) {
-			CommonToken ct = (CommonToken)t;
 			copy.add(new CommonToken(t));
 		}
 		return copy;
@@ -364,6 +343,7 @@ public static double weightedL0_Distance(FeatureMetaData[] featureTypes, int[] A
 		for (int i=0; i<A.length; i++) {
 			if ( featureTypes[i].type==FeatureType.TOKEN ||
 				featureTypes[i].type==FeatureType.RULE  ||
+				featureTypes[i].type==FeatureType.INT  ||
 				featureTypes[i].type==FeatureType.BOOL
 				)
 			{

java/src/org/antlr/codebuff/gui/GUIController.java

@@ -16,7 +16,6 @@
 import java.awt.*;
 import java.util.List;
 
-import static javax.swing.JFrame.EXIT_ON_CLOSE;
 import static javax.swing.text.DefaultHighlighter.DefaultHighlightPainter;
 
 public class GUIController {
@@ -71,7 +70,7 @@ public void show() throws Exception {
 		scope.injectWSConsole.setFont(docFont);
 
 		JFrame frame = new JFrame("CodeBuff Scope");
-		frame.setDefaultCloseOperation(EXIT_ON_CLOSE);
+		frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
 		frame.getContentPane().add(scope.$$$getRootComponent$$$(), BorderLayout.CENTER);
 
 		scope.getFormattedTextPane().addCaretListener(new HighlightTokenListener());